diff --git a/CLAUDE.md b/CLAUDE.md index 7d9a6367f18..84d2efaab6a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,6 +17,8 @@ Same thing for bug fixes. The tests should make it so that this specific bug can `tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_.py` if you're the first test there). One focused regression test beats many shallow ones +End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md` + When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose Always use @.github/pull_request_template.md as a guide for your PR body diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 12aea6bbc25..4037f324393 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -9,15 +9,18 @@ When contributing to this directory, please first discuss the change you wish to ## Setup -The suites run against a live proxy, so bring one up first. `docker-compose.yml` here starts that proxy with its Postgres and Redis, serving `gateway/litellm-config.yml`; add any model, pricing override, or guardrail your test needs to that file and read it back in the test rather than hardcoding values. `gateway/` holds proxy configuration only, so never put tests there +The suites run against a live proxy, so bring one up first. `docker-compose.yml` here starts that proxy with a throwaway Postgres and Redis; `docker compose down -v` resets everything, so no state leaks between runs. The proxy config is inlined in the compose file under `configs`, prewired with example models (`gpt-5.5`, `claude-haiku-4-5`, `gemini-2.5-flash`, `openai-text-embedding-3-small`) whose keys come from your `.env`. If your test needs another model, a pricing override, or a guardrail declared up front, add it to that inline config and read it back in the test rather than hardcoding values ## Running the tests locally -1. Create a .env file and add provider keys: +1. Create a `.env` file in this directory with the provider keys the example models use: + ```bash OPENAI_API_KEY="sk-..." ANTHROPIC_API_KEY="sk-..." - + GEMINI_API_KEY="..." + ``` + 2. Bring the stack up from this directory: ```bash @@ -123,7 +126,17 @@ Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure cover Before you push -- Run basedpyright over your changes; the harness is fully typed and new code must not add `Any` or widen the budgets -- Bring the stack up with docker-compose from this directory and run your suite locally against it, so you exercise the same skip-vs-fail path CI does -- Use the config at `tests/e2e/gateway/litellm-config.yml` if your feature needs a model, pricing override, guardrail, or other proxy setting declared up front; add the deployment there and read it back in the test rather than hardcoding values -- Capture screenshots of the tests passing and attach them to the PR as proof of fix +1. Run basedpyright over your changes; the harness is fully typed and new code must not add `Any` or widen the budgets + +2. Add the models your test needs to the inline config in `docker-compose.yml` + +3. Bring the stack up and run your suite against it: + + ```bash + docker compose up -d + uv run pytest tests/e2e// -v + ``` + +4. Capture screenshots of the test run and attach them to the PR as proof + +5. If a test fails because it surfaced a real issue in the product, flag that explicitly in the PR rather than reworking the test until it passes diff --git a/tests/e2e/docker-compose.yml b/tests/e2e/docker-compose.yml new file mode 100644 index 00000000000..2de38ca5c91 --- /dev/null +++ b/tests/e2e/docker-compose.yml @@ -0,0 +1,87 @@ +# local setup to run e2e tests +configs: + litellm_config: + content: | + general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL + store_prompts_in_spend_logs: true + + litellm_settings: + drop_params: true + num_retries: 3 + request_timeout: 600 + cache: true + cache_params: + type: redis + host: redis + port: 6379 + + router_settings: + routing_strategy: simple-shuffle + num_retries: 3 + allowed_fails: 5 + cooldown_time: 30 + fallbacks: + - gemini-2.5-flash: ["gpt-5.5", "claude-haiku-4-5"] + + model_list: + - model_name: gpt-5.5 + litellm_params: + model: openai/gpt-5.5 + api_key: os.environ/OPENAI_API_KEY + + - model_name: claude-haiku-4-5 + litellm_params: + model: anthropic/claude-haiku-4-5 + api_key: os.environ/ANTHROPIC_API_KEY + + - model_name: gemini-2.5-flash + litellm_params: + model: gemini/gemini-2.5-flash + api_key: os.environ/GEMINI_API_KEY + + - model_name: openai-text-embedding-3-small + litellm_params: + model: openai/text-embedding-3-small + api_key: os.environ/OPENAI_API_KEY + +services: + litellm: + image: ghcr.io/berriai/litellm:main-latest + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + env_file: .env + environment: + LITELLM_MASTER_KEY: sk-1234 + DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm + ports: + - "4000:4000" + configs: + - source: litellm_config + target: /app/config.yaml + command: ["--config", "/app/config.yaml", "--port", "4000"] + +# throwaway db + db: + image: postgres:16 + environment: + POSTGRES_USER: litellm + POSTGRES_PASSWORD: litellm + POSTGRES_DB: litellm + healthcheck: + test: ["CMD-SHELL", "pg_isready -U litellm"] + interval: 3s + timeout: 3s + retries: 20 + + redis: + image: redis:7 + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 3s + retries: 20 diff --git a/tests/e2e/gateway/litellm-config.yml b/tests/e2e/gateway/litellm-config.yml deleted file mode 100644 index 3e5237b004b..00000000000 --- a/tests/e2e/gateway/litellm-config.yml +++ /dev/null @@ -1,287 +0,0 @@ -# This default config file aims to support most popular model providers out of the box - -#In general, the model name used by the client will be the same as the ones from the provider (For example, you will use "anthropic.claude-3-5-sonnet-20240620-v1:0" when you're calling LiteLLM just like you would when calling Amazon Bedrock directly) -#In the case where there are model name conflicts, a prefix will be used (For example, the Azure and the openAI model names conflict, so when you are using Azure, you will use "azure/gpt-4o-realtime-preview-2024-10-01") - -#Some model providers require additional user-specific configuration (such as Azure which requires you to specify your own api_base with your resource name, and your api_version). -#In this case, the provider is commented out, and you should uncomment it and provide your specific info - -#For more detailed information about each provider, refer to the docs: https://docs.litellm.ai/docs/providers - -#If you are not interested in a particular provider, just remove it from your config.yaml, and redeploy, and it will no longer show up in your LiteLLM deployment - -#If a particular provider is not working, double check your .env file, and make sure you have provided a valid api key for that provider, and then redeploy - -#Full details on guardrails here: https://docs.litellm.ai/docs/proxy/guardrails/bedrock -general_settings: - store_prompts_in_spend_logs: true - master_key: os.environ/LITELLM_MASTER_KEY - proxy_batch_write_at: 60 - database_connection_pool_limit: 10 - # disable_error_logs: True - forward_client_headers_to_llm_api: false - maximum_spend_logs_retention_period: "60d" # GSE-13389: Cleanup logs older than 60 days - maximum_spend_logs_cleanup_cron: "0 1 * * *" # 01:00 UTC daily = 18:00 PDT - database_url: os.environ/DATABASE_URL - control_plane_url: os.environ/CONTROL_PLANE_URL - alerts: ["email"] - - proxy_budget_rescheduler_min_time: 15 - proxy_budget_rescheduler_max_time: 20 - -# fallbacks: [{"gpt-4": ["anthropic.claude-3-5-sonnet-20240620-v1:0"]}] #Configure fallbacks for context window exeeded errors (In this example, we will fall back to Claude Sonnet if over 8000 tokens, which is gpt-4's limit) - # default_fallbacks: ["anthropic.claude-3-haiku-20240307-v1:0"] #Configure fallbacks for any error for every model (the above fallback configurations override this one) -# environment_variables: -# STORE_MODEL_IN_DB: 'True' -# LITELLM_LOG: "DEBUG" -litellm_settings: - drop_params: True - # Spend counters inherit this as their Redis TTL, so an idle counter goes cold and - # the next request reseeds it from the DB; kept short to exercise the cross-pod - # reseed path in test_spend_counter_reseed_e2e. Response-cache writes pass their own - # ttl and are unaffected. - default_redis_ttl: 20 - request_timeout: 600 - num_retries: 3 - json_logs: true - store_audit_logs: True - cache: true - cache_params: - type: redis - host: redis - port: 6379 - password: os.environ/REDIS_PASSWORD - namespace: litellm.caching - ttl: 16600 - # max_budget: 1000000000.0 # (float) sets max budget in dollars across the entire proxy across all API keys. Note, the budget does not apply to the master key. That is the only exception. - # budget_duration: 1mo # (str) frequency of budget reset - You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo"). - # max_internal_user_budget: 1000000000.0 # (float) sets default budget in dollars for each internal user. (Doesn't apply to Admins. Doesn't apply to Teams. Doesn't apply to master key) - # internal_user_budget_duration: "1mo" # (str) frequency of budget reset - You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo"). - # success_callback: ["s3_v2"] - # failure_callback: ["s3_v2"] - # service_callback: ["datadog"] - callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel"] - require_auth_for_metrics_endpoint: false - #type: redis-semantic - #similarity_threshold: 0.8 # similarity threshold for semantic cache - #redis_semantic_cache_embedding_model: text-embedding-ada-002 # only works with text-embedding-ada-002 for now... https://github.com/BerriAI/litellm/issues/4001 - -router_settings: - routing_strategy: simple-shuffle - num_retries: 3 - allowed_fails: 5 - cooldown_time: 30 - # When gemini deployments are exhausted (provider 429 / auth), cross over to - # working models. Exercised by tests/e2e/router/test_rate_limiter.py. - fallbacks: - - gemini-2.5-flash: ["gpt-5.5", "claude-haiku-4-5"] - -#ttl: Optional[float] -#default_in_memory_ttl: Optional[float] -#default_in_redis_ttl: Optional[float] - -model_list: - - model_name: gpt-5.5 - litellm_params: - model: openai/gpt-5.5 - api_key: os.environ/OPENAI_API_KEY - - - model_name: claude-haiku-4-5 - litellm_params: - model: anthropic/claude-haiku-4-5 - api_key: os.environ/ANTHROPIC_API_KEY - - # Same underlying model via Vertex AI — distinct routing/auth path - # # (service-account JSON), so it gets its own model_name. - - model_name: gemini-2.5-flash-vertex - litellm_params: - model: vertex_ai/gemini-2.5-flash - vertex_project: os.environ/VERTEXAI_PROJECT - vertex_location: us-central1 - vertex_credentials: os.environ/VERTEXAI_CREDENTIALS - - - model_name: gemini-2.5-flash - litellm_params: - model: gemini/gemini-2.5-flash - api_key: os.environ/GEMINI_API_KEY - - # load balancing to a different deployment, if gemini gets rate limited. - - model_name: gemini-2.5-flash - litellm_params: - model: gemini/gemini-2.5-flash - api_key: os.environ/GEMINI_API_KEY - - # Custom per-token pricing exercised by llm_translation/test_custom_pricing_e2e.py. - # Rates deliberately exceed canonical gemini-2.5-flash (input 3e-7 / output 2.5e-6) - # so an override that is ignored or under-applied reports spend at the base rate - # and fails that test. The test reads these same rates back from this file. - - model_name: custom-priced-flash - litellm_params: - model: gemini/gemini-2.5-flash - api_key: os.environ/GEMINI_API_KEY - input_cost_per_token: 0.00005 - output_cost_per_token: 0.0001 - - # embedding models - - model_name: openai-text-embedding-3-small - litellm_params: - model: openai/text-embedding-3-small - api_key: os.environ/OPENAI_API_KEY - - - model_name: gemini-2-embedding - litellm_params: - model: gemini/gemini-2-embedding - api_key: os.environ/GEMINI_API_KEY - - - model_name: openai-realtime - litellm_params: - model: openai/gpt-realtime - api_key: os.environ/OPENAI_API_KEY - model_info: - mode: realtime - - - model_name: azure-realtime - litellm_params: - model: azure/gpt-realtime-2 - api_key: os.environ/AZURE_API_KEY - api_base: os.environ/AZURE_API_BASE - api_version: "2025-08-28" - realtime_protocol: GA # Possible values: "GA"/ "v1", "beta" - model_info: - mode: realtime - - - model_name: gemini-realtime - litellm_params: - model: gemini/gemini-3.1-flash-live-preview - api_key: os.environ/GEMINI_API_KEY - model_info: - mode: realtime - - - model_name: vertex-realtime - litellm_params: - model: vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025 - vertex_project: os.environ/VERTEXAI_PROJECT - vertex_location: us-central1 - vertex_credentials: os.environ/VERTEXAI_CREDENTIALS - model_info: - mode: realtime - - - model_name: bedrock-realtime - litellm_params: - model: bedrock/amazon.nova-sonic-v1:0 - aws_region_name: us-east-1 - model_info: - mode: realtime - - - model_name: xai-realtime - litellm_params: - model: xai/grok-voice-latest - api_key: os.environ/XAI_API_KEY - model_info: - mode: realtime - - model_name: rust-ocr-mistral - litellm_params: - model: mistral/mistral-ocr-latest - api_key: os.environ/MISTRAL_API_KEY - - - model_name: rust-ocr-azure-ai - litellm_params: - model: azure_ai/mistral-document-ai-2505 - api_base: os.environ/AZURE_API_BASE - api_key: os.environ/AZURE_API_KEY - - - model_name: rust-ocr-azure-document-intelligence - litellm_params: - model: azure_ai/doc-intelligence/prebuilt-layout - api_base: os.environ/AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT - api_key: os.environ/AZURE_DOCUMENT_INTELLIGENCE_API_KEY - - - model_name: rust-ocr-vertex-mistral - litellm_params: - model: vertex_ai/mistral-ocr-2505 - vertex_project: os.environ/VERTEXAI_PROJECT - vertex_location: us-central1 - - - model_name: rust-ocr-vertex-deepseek - litellm_params: - model: vertex_ai/deepseek-ocr-maas - vertex_project: os.environ/VERTEXAI_PROJECT - vertex_location: us-central1 - - # batch models exercised by tests/e2e/batches/ - - model_name: openai-batch - litellm_params: - model: openai/gpt-4o-mini - api_key: os.environ/OPENAI_API_KEY - model_info: - mode: batch - - - model_name: azure-batch - litellm_params: - model: azure/gpt-4.1-mini-batch - api_base: os.environ/AZURE_API_BASE - api_key: os.environ/AZURE_API_KEY - api_version: "2024-07-01-preview" - model_info: - mode: batch - - - model_name: vertex-batch - litellm_params: - model: vertex_ai/gemini-2.5-flash - vertex_project: os.environ/VERTEXAI_PROJECT - vertex_location: us-central1 - vertex_credentials: os.environ/VERTEXAI_CREDENTIALS - bucket_name: os.environ/GCS_BUCKET_NAME - model_info: - mode: batch - - - model_name: bedrock-batch - litellm_params: - model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 - s3_bucket_name: os.environ/AWS_BATCH_S3_BUCKET - s3_region_name: us-west-2 - s3_access_key_id: os.environ/AWS_ACCESS_KEY_ID - s3_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY - aws_batch_role_arn: os.environ/AWS_BATCH_ROLE_ARN - model_info: - mode: batch - -files_settings: - - custom_llm_provider: openai - api_key: os.environ/OPENAI_API_KEY - - custom_llm_provider: azure - api_base: os.environ/AZURE_API_BASE - api_key: os.environ/AZURE_API_KEY - api_version: "2024-07-01-preview" - - custom_llm_provider: vertex_ai - vertex_project: os.environ/VERTEXAI_PROJECT - vertex_location: us-central1 - vertex_credentials: os.environ/VERTEXAI_CREDENTIALS - bucket_name: os.environ/GCS_BUCKET_NAME - - -mcp_servers: - deepwiki_mcp: - url: "https://mcp.deepwiki.com/mcp" - auth_type: none - description: "just a test" - - atlassian: - url: "https://mcp.atlassian.com/v1/mcp" - auth_type: oauth2 - authorization_url: https://auth.atlassian.com/authorize - - -guardrails: - - guardrail_name: "presidio-pii" - litellm_params: - guardrail: presidio - mode: pre_call - presidio_analyzer_api_base: os.environ/PRESIDIO_ANALYZER_API_BASE - presidio_anonymizer_api_base: os.environ/PRESIDIO_ANONYMIZER_API_BASE - default_on: false - pii_entities_config: - EMAIL_ADDRESS: BLOCK - CREDIT_CARD: BLOCK - US_SSN: BLOCK - PHONE_NUMBER: BLOCK