# This default config file aims to support most popular model providers out of the box #In general, the model name used by the client will be the same as the ones from the provider (For example, you will use "anthropic.claude-3-5-sonnet-20240620-v1:0" when you're calling LiteLLM just like you would when calling Amazon Bedrock directly) #In the case where there are model name conflicts, a prefix will be used (For example, the Azure and the openAI model names conflict, so when you are using Azure, you will use "azure/gpt-4o-realtime-preview-2024-10-01") #Some model providers require additional user-specific configuration (such as Azure which requires you to specify your own api_base with your resource name, and your api_version). #In this case, the provider is commented out, and you should uncomment it and provide your specific info #For more detailed information about each provider, refer to the docs: https://docs.litellm.ai/docs/providers #If you are not interested in a particular provider, just remove it from your config.yaml, and redeploy, and it will no longer show up in your LiteLLM deployment #If a particular provider is not working, double check your .env file, and make sure you have provided a valid api key for that provider, and then redeploy #Full details on guardrails here: https://docs.litellm.ai/docs/proxy/guardrails/bedrock general_settings: store_prompts_in_spend_logs: true master_key: os.environ/LITELLM_MASTER_KEY proxy_batch_write_at: 60 database_connection_pool_limit: 10 # disable_error_logs: True forward_client_headers_to_llm_api: false maximum_spend_logs_retention_period: "60d" # GSE-13389: Cleanup logs older than 60 days maximum_spend_logs_cleanup_cron: "0 1 * * *" # 01:00 UTC daily = 18:00 PDT database_url: os.environ/DATABASE_URL control_plane_url: os.environ/CONTROL_PLANE_URL alerts: ["email"] proxy_budget_rescheduler_min_time: 15 proxy_budget_rescheduler_max_time: 20 # fallbacks: [{"gpt-4": ["anthropic.claude-3-5-sonnet-20240620-v1:0"]}] #Configure fallbacks for context window exeeded errors (In this example, we will fall back to Claude Sonnet if over 8000 tokens, which is gpt-4's limit) # default_fallbacks: ["anthropic.claude-3-haiku-20240307-v1:0"] #Configure fallbacks for any error for every model (the above fallback configurations override this one) # environment_variables: # STORE_MODEL_IN_DB: 'True' # LITELLM_LOG: "DEBUG" litellm_settings: drop_params: True # Spend counters inherit this as their Redis TTL, so an idle counter goes cold and # the next request reseeds it from the DB; kept short to exercise the cross-pod # reseed path in test_spend_counter_reseed_e2e. Response-cache writes pass their own # ttl and are unaffected. default_redis_ttl: 20 request_timeout: 600 num_retries: 3 json_logs: true store_audit_logs: True cache: true cache_params: type: redis host: redis port: 6379 password: os.environ/REDIS_PASSWORD namespace: litellm.caching ttl: 16600 # max_budget: 1000000000.0 # (float) sets max budget in dollars across the entire proxy across all API keys. Note, the budget does not apply to the master key. That is the only exception. # budget_duration: 1mo # (str) frequency of budget reset - You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo"). # max_internal_user_budget: 1000000000.0 # (float) sets default budget in dollars for each internal user. (Doesn't apply to Admins. Doesn't apply to Teams. Doesn't apply to master key) # internal_user_budget_duration: "1mo" # (str) frequency of budget reset - You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo"). # success_callback: ["s3_v2"] # failure_callback: ["s3_v2"] # service_callback: ["datadog"] callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel"] require_auth_for_metrics_endpoint: false #type: redis-semantic #similarity_threshold: 0.8 # similarity threshold for semantic cache #redis_semantic_cache_embedding_model: text-embedding-ada-002 # only works with text-embedding-ada-002 for now... https://github.com/BerriAI/litellm/issues/4001 router_settings: routing_strategy: simple-shuffle num_retries: 3 allowed_fails: 5 cooldown_time: 30 # When gemini deployments are exhausted (provider 429 / auth), cross over to # working models. Exercised by tests/e2e/router/test_rate_limiter.py. fallbacks: - gemini-2.5-flash: ["gpt-5.5", "claude-haiku-4-5"] #ttl: Optional[float] #default_in_memory_ttl: Optional[float] #default_in_redis_ttl: Optional[float] model_list: - model_name: gpt-5.5 litellm_params: model: openai/gpt-5.5 api_key: os.environ/OPENAI_API_KEY - model_name: claude-haiku-4-5 litellm_params: model: anthropic/claude-haiku-4-5 api_key: os.environ/ANTHROPIC_API_KEY # Same underlying model via Vertex AI — distinct routing/auth path # # (service-account JSON), so it gets its own model_name. - model_name: gemini-2.5-flash-vertex litellm_params: model: vertex_ai/gemini-2.5-flash vertex_project: os.environ/VERTEXAI_PROJECT vertex_location: us-central1 vertex_credentials: os.environ/VERTEXAI_CREDENTIALS - model_name: gemini-2.5-flash litellm_params: model: gemini/gemini-2.5-flash api_key: os.environ/GEMINI_API_KEY # load balancing to a different deployment, if gemini gets rate limited. - model_name: gemini-2.5-flash litellm_params: model: gemini/gemini-2.5-flash api_key: os.environ/GEMINI_API_KEY # Custom per-token pricing exercised by llm_translation/test_custom_pricing_e2e.py. # Rates deliberately exceed canonical gemini-2.5-flash (input 3e-7 / output 2.5e-6) # so an override that is ignored or under-applied reports spend at the base rate # and fails that test. The test reads these same rates back from this file. - model_name: custom-priced-flash litellm_params: model: gemini/gemini-2.5-flash api_key: os.environ/GEMINI_API_KEY input_cost_per_token: 0.00005 output_cost_per_token: 0.0001 # embedding models - model_name: openai-text-embedding-3-small litellm_params: model: openai/text-embedding-3-small api_key: os.environ/OPENAI_API_KEY - model_name: gemini-2-embedding litellm_params: model: gemini/gemini-2-embedding api_key: os.environ/GEMINI_API_KEY - model_name: openai-realtime litellm_params: model: openai/gpt-realtime api_key: os.environ/OPENAI_API_KEY model_info: mode: realtime - model_name: azure-realtime litellm_params: model: azure/gpt-realtime-2 api_key: os.environ/AZURE_API_KEY api_base: os.environ/AZURE_API_BASE api_version: "2025-08-28" realtime_protocol: GA # Possible values: "GA"/ "v1", "beta" model_info: mode: realtime - model_name: gemini-realtime litellm_params: model: gemini/gemini-3.1-flash-live-preview api_key: os.environ/GEMINI_API_KEY model_info: mode: realtime - model_name: vertex-realtime litellm_params: model: vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025 vertex_project: os.environ/VERTEXAI_PROJECT vertex_location: us-central1 vertex_credentials: os.environ/VERTEXAI_CREDENTIALS model_info: mode: realtime - model_name: bedrock-realtime litellm_params: model: bedrock/amazon.nova-sonic-v1:0 aws_region_name: us-east-1 model_info: mode: realtime - model_name: xai-realtime litellm_params: model: xai/grok-voice-latest api_key: os.environ/XAI_API_KEY model_info: mode: realtime - model_name: rust-ocr-mistral litellm_params: model: mistral/mistral-ocr-latest api_key: os.environ/MISTRAL_API_KEY - model_name: rust-ocr-azure-ai litellm_params: model: azure_ai/mistral-document-ai-2505 api_base: os.environ/AZURE_API_BASE api_key: os.environ/AZURE_API_KEY - model_name: rust-ocr-azure-document-intelligence litellm_params: model: azure_ai/doc-intelligence/prebuilt-layout api_base: os.environ/AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT api_key: os.environ/AZURE_DOCUMENT_INTELLIGENCE_API_KEY - model_name: rust-ocr-vertex-mistral litellm_params: model: vertex_ai/mistral-ocr-2505 vertex_project: os.environ/VERTEXAI_PROJECT vertex_location: us-central1 - model_name: rust-ocr-vertex-deepseek litellm_params: model: vertex_ai/deepseek-ocr-maas vertex_project: os.environ/VERTEXAI_PROJECT vertex_location: us-central1 mcp_servers: deepwiki_mcp: url: "https://mcp.deepwiki.com/mcp" auth_type: none description: "just a test" atlassian: url: "https://mcp.atlassian.com/v1/mcp" auth_type: oauth2 authorization_url: https://auth.atlassian.com/authorize guardrails: - guardrail_name: "presidio-pii" litellm_params: guardrail: presidio mode: pre_call presidio_analyzer_api_base: os.environ/PRESIDIO_ANALYZER_API_BASE presidio_anonymizer_api_base: os.environ/PRESIDIO_ANONYMIZER_API_BASE default_on: false pii_entities_config: EMAIL_ADDRESS: BLOCK CREDIT_CARD: BLOCK US_SSN: BLOCK PHONE_NUMBER: BLOCK