diff --git a/.circleci/config.yml b/.circleci/config.yml index bfca8310c0e..a93d62d0c39 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -473,7 +473,7 @@ jobs: command: | python3 tests/windows_tests/check_windows_wheel_install.py --lengths-only - local_testing_part1: + auth_ui_unit_tests: docker: - &python312_image image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c @@ -489,219 +489,6 @@ jobs: working_directory: ~/project environment: &sidecar_database_url DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" - parallelism: 4 - steps: - - checkout - - skip_if_unrelated_changes - - setup_google_dns - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - install_uv - - install_rust - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ~/.cache/uv - key: v1-uv-cache-{{ checksum "uv.lock" }} - - save_cargo_target - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - # Run pytest and generate JUnit XML report - - run: - name: Run tests (Part 1 - A-M) - command: | - mkdir test-results - - # Discover test files (A-M) - TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py") - - echo "$TEST_FILES" | circleci tests run \ - --split-by=timings \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \ - -vv \ - --cov=./litellm --cov=./enterprise/litellm_enterprise \ - --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=20 \ - -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ - -n 4 \ - --timeout=300 \ - --timeout_method=thread" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - # When CI reruns only the failed tests, a parallel node can receive - # zero tests and pytest never writes coverage. Emit empty placeholders - # so persist_to_workspace and the downstream coverage combine stay green. - if [ -f coverage.xml ]; then - mv coverage.xml local_testing_part1_coverage.xml - mv .coverage local_testing_part1_coverage - else - touch local_testing_part1_coverage.xml local_testing_part1_coverage - fi - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - local_testing_part1_coverage.xml - - local_testing_part1_coverage - local_testing_part2: - docker: - - *python312_image - - *postgres16_sidecar - working_directory: ~/project - environment: *sidecar_database_url - parallelism: 4 - steps: - - checkout - - skip_if_unrelated_changes - - setup_google_dns - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - install_uv - - install_rust - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ~/.cache/uv - key: v1-uv-cache-{{ checksum "uv.lock" }} - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - # Run pytest and generate JUnit XML report - - run: - name: Run tests (Part 2 - N-Z) - command: | - mkdir test-results - - # Discover test files (N-Z) - TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py") - - echo "$TEST_FILES" | circleci tests run \ - --split-by=timings \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \ - -vv \ - --cov=./litellm --cov=./enterprise/litellm_enterprise \ - --cov-report=xml \ - --junitxml=test-results/junit.xml \ - --durations=20 \ - -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ - -n 4 \ - --timeout=300 \ - --timeout_method=thread" - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - # When CI reruns only the failed tests, a parallel node can receive - # zero tests and pytest never writes coverage. Emit empty placeholders - # so persist_to_workspace and the downstream coverage combine stay green. - if [ -f coverage.xml ]; then - mv coverage.xml local_testing_part2_coverage.xml - mv .coverage local_testing_part2_coverage - else - touch local_testing_part2_coverage.xml local_testing_part2_coverage - fi - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - local_testing_part2_coverage.xml - - local_testing_part2_coverage - langfuse_logging_unit_tests: - docker: - - *python312_image - - *postgres16_sidecar - working_directory: ~/project - environment: *sidecar_database_url - resource_class: medium - - steps: - - checkout - - skip_if_unrelated_changes - - setup_google_dns - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - install_uv - - install_rust - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ~/.cache/uv - key: v1-uv-cache-{{ checksum "uv.lock" }} - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - - run: - name: Run prisma ./docker/entrypoint.sh - command: | - set +e - chmod +x docker/entrypoint.sh - ./docker/entrypoint.sh - set -e - - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \ - -v \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -k \"langfuse\"" - no_output_timeout: 15m - # Store test results - - store_test_results: - path: test-results - auth_ui_unit_tests: - docker: - - *python312_image - - *postgres16_sidecar - working_directory: ~/project - environment: *sidecar_database_url steps: - checkout @@ -759,59 +546,6 @@ jobs: - auth_ui_unit_tests_coverage.xml - auth_ui_unit_tests_coverage - litellm_router_testing: # Runs all tests with the "router" keyword - docker: - - *python312_image - working_directory: ~/project - resource_class: large - parallelism: 1 - environment: - FAKE_OPENAI_API_BASE: http://127.0.0.1:8190 - steps: - - checkout - - skip_if_unrelated_changes - - setup_google_dns - - install_uv - - install_rust - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - save_cache: - paths: - - ~/.cache/uv - key: v1-uv-cache-{{ checksum "uv.lock" }} - - start_fake_openai_endpoint - # Run pytest and generate JUnit XML report - - setup_litellm_enterprise_pip - - run: - name: Run tests - command: | - # On a "rerun failed tests" build a parallel node can receive no - # tests, so the test command never creates test-results. Pre-create it - # so store_test_results doesn't fail the node on a missing path. - mkdir -p test-results - - TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") - - echo "$TEST_FILES" | circleci tests run \ - --split-by=timings \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \ - -v \ - -k 'router' \ - -n 4 \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - --timeout=300 --timeout_method=thread" - no_output_timeout: 15m - # Store test results - - store_test_results: - path: test-results - llm_translation_testing: docker: - *python312_image @@ -2222,7 +1956,7 @@ jobs: - run: name: Combine Coverage command: | - uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage google_generate_content_endpoint_coverage litellm_utils_coverage + uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage pass_through_unit_tests_coverage batches_coverage google_generate_content_endpoint_coverage litellm_utils_coverage uv tool run --from 'coverage[toml]==7.10.6' coverage xml - codecov/upload: file: ./coverage.xml @@ -3306,10 +3040,6 @@ workflows: equal: ["", << pipeline.parameters.routing_parity_base >>] jobs: - provider_replay_harness - - local_testing_part1 - - local_testing_part2 - - langfuse_logging_unit_tests - - litellm_router_testing - auth_ui_unit_tests - build_docker_database_image - e2e_ui_testing @@ -3359,9 +3089,6 @@ workflows: - image_gen_testing - logging_testing - audio_testing - - langfuse_logging_unit_tests - - local_testing_part1 - - local_testing_part2 - db_migration_disable_update_check: requires: - build_docker_database_image diff --git a/.github/ci-coverage-allowlist.yml b/.github/ci-coverage-allowlist.yml index b3217059e81..4ac8224337f 100644 --- a/.github/ci-coverage-allowlist.yml +++ b/.github/ci-coverage-allowlist.yml @@ -18,9 +18,8 @@ test_paths: paths: - tests/rust-python-harness - reason: >- - Live-provider caching cases in tests/local_testing that remain outside CI. Jobs that - glob that directory either deselect them (local_testing_part1 and part2 carry `-k "... and - not caching and not cache"`) or keep only another keyword (langfuse, router, assistants). + Live-provider caching cases in tests/local_testing that remain outside CI. No CircleCI job + globs that directory anymore, and the legacy jobs that did always deselected these files. The two Redis Cluster IAM authentication tests moved to tests/integration/sdk and remain selected by node ID in redis_compat, where redis-server is installed. The five keyless files now run in the caching-local shard, leaving live cases in these two. @@ -30,14 +29,6 @@ test_paths: paths: - tests/local_testing/test_disk_cache_unit_tests.py - tests/local_testing/test_gcs_cache_unit_tests.py - - reason: >- - Mixed file that no job has ever run in full. Every CircleCI job that globs - tests/local_testing deselects it by name ("caching") or keeps only another keyword, so only - its router test ran, and that test now lives in tests/unit/router_utils/pre_call_checks. - Its two offline tests now live in tests/unit/llms/anthropic/chat, and the five left all need - ANTHROPIC_API_KEY or Vertex credentials - paths: - - tests/local_testing/test_anthropic_prompt_caching.py - reason: >- The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than from a pull request; it needs a live gateway and provider credentials no PR job holds diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index a784d51be1c..4477b9584aa 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -37,6 +37,8 @@ from e2e_config import ( PROVIDER_EDGE_HOST_OPT_IN_ENV, PROXY_BASE_URL, REDIS_CHAOS_OPT_IN_ENV, + SAGEMAKER_NOVA2_LITE_OPT_IN_ENV, + SAGEMAKER_NOVA_OPT_IN_ENV, SECRET_MANAGER_OPT_IN_ENV, WEEKLY_ANOMALY_OPT_IN_ENV, unique_marker, @@ -75,6 +77,8 @@ OPT_IN_MARKERS: Final = MappingProxyType( "otel_v2": OTEL_V2_OPT_IN_ENV, "otel_tls": OTEL_TLS_OPT_IN_ENV, "secret_manager": SECRET_MANAGER_OPT_IN_ENV, + "sagemaker_nova": SAGEMAKER_NOVA_OPT_IN_ENV, + "sagemaker_nova2_lite": SAGEMAKER_NOVA2_LITE_OPT_IN_ENV, } ) @@ -192,6 +196,16 @@ def pytest_configure(config: pytest.Config) -> None: "secret_manager: needs a proxy booted from gateway/secret_manager__ci_config.yml against that live " "secret manager; deselected unless E2E_SECRET_MANAGER names the backend (see secret_manager/secret_backends.py)", ) + config.addinivalue_line( + "markers", + "sagemaker_nova: needs a deployed SageMaker Nova endpoint the proxy's AWS credentials can invoke; " + "deselected unless SAGEMAKER_NOVA_ENDPOINT names it", + ) + config.addinivalue_line( + "markers", + "sagemaker_nova2_lite: needs a deployed SageMaker Nova 2 Lite endpoint the proxy's AWS credentials can invoke; " + "deselected unless SAGEMAKER_NOVA2_LITE_ENDPOINT names it", + ) def pytest_sessionstart(session: pytest.Session) -> None: diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index cd41a4548ee..b11ae907cfb 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -32,6 +32,7 @@ - {id: llm.chat_completions.bedrock_converse.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Bedrock vision (Anthropic/Nova)"} - {id: llm.chat_completions.bedrock_converse.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: bedrock_converse, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Anthropic-on-Bedrock caching"} - {id: llm.chat_completions.bedrock_converse.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: bedrock_converse, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Anthropic thinking on Bedrock"} +- {id: llm.chat_completions.bedrock_converse.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: bedrock_converse, capability: structured_output, streaming: nonstream, assertions: [works], source: "llms/bedrock/chat/converse_handler.py", rationale: "Bedrock Claude accepts a Pydantic-derived response schema"} - {id: llm.chat_completions.bedrock_converse.response_headers.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: response_headers, streaming: nonstream, assertions: [works], source: "llms/bedrock/chat/converse_handler.py:248", rationale: "Bedrock request ids must surface as llm_provider-* response headers on /chat/completions so callers can correlate calls with AWS-side logs (#37003)", fail_before_fix: proven} - {id: llm.chat_completions.bedrock_converse.response_headers.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: response_headers, streaming: stream, assertions: [works], source: "llms/bedrock/chat/converse_handler.py:154", rationale: "The llm_provider-* headers must also surface on streaming /chat/completions, where CustomStreamWrapper carries them instead of the nonstream setter"} - {id: llm.chat_completions.bedrock_converse.batch_deployment.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: batch_deployment, streaming: nonstream, assertions: [works], source: "types/utils.py bedrock_batch_litellm_params", rationale: "A deployment carrying the documented batch-only S3 keys (s3_access_key_id, s3_secret_access_key, s3_encryption_key_id) must still serve ordinary chat; unregistered keys fall into optional_params and are forwarded as additionalModelRequestFields, which Bedrock 400s and which puts the S3 secret in the request body and debug log (LIT-8290)", fail_before_fix: proven} @@ -47,8 +48,13 @@ - {id: llm.chat_completions.vertex.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Streaming over Vertex"} - {id: llm.chat_completions.vertex.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vertex Gemini function_calling"} +- {id: llm.chat_completions.vertex.web_search.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: vertex, capability: web_search, streaming: stream, assertions: [works], source: "llms/vertex_ai", rationale: "Vertex Gemini streams Google Search grounding metadata"} - {id: llm.chat_completions.vertex.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Gemini vision"} - {id: llm.chat_completions.vertex.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: vertex, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vertex Gemini prompt caching"} +- {id: llm.chat_completions.sagemaker.vision.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: sagemaker, capability: vision, streaming: nonstream, assertions: [works], source: "llms/sagemaker/nova", rationale: "SageMaker Nova accepts image input"} +- {id: llm.chat_completions.sagemaker.top_k.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: sagemaker, capability: top_k, streaming: nonstream, assertions: [works], source: "llms/sagemaker/nova", rationale: "SageMaker Nova accepts its top_k request parameter"} +- {id: llm.chat_completions.sagemaker.logprobs.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: sagemaker, capability: logprobs, streaming: nonstream, assertions: [works], source: "llms/sagemaker/nova", rationale: "SageMaker Nova returns log probability details"} +- {id: llm.chat_completions.sagemaker.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: sagemaker, capability: thinking, streaming: nonstream, assertions: [works], source: "llms/sagemaker/nova", rationale: "SageMaker Nova 2 Lite accepts reasoning effort"} - {id: llm.chat_completions.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Azure OpenAI deployments"} - {id: llm.chat_completions.azure_openai.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: azure_openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Azure OpenAI function_calling"} - {id: llm.chat_completions.azure_foundry.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: azure_foundry, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "Azure Foundry (azure_ai); newer, smoke"} diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index d09199ed138..d10035d6359 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -8,6 +8,7 @@ - {id: llm.embeddings.bedrock.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "llms/bedrock/embed/embedding.py", rationale: "Bedrock Titan embeddings"} - {id: llm.embeddings.vertex.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "vertex_embeddings/embedding_handler.py", rationale: "Vertex embeddings"} - {id: llm.embeddings.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "llms/cohere/embed/handler.py", rationale: "Cohere embeddings"} +- {id: llm.embeddings.cohere.vision.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: cohere, capability: vision, streaming: nonstream, assertions: [works], source: "llms/cohere/embed/handler.py", rationale: "Cohere image embeddings report image token usage"} - {id: llm.embeddings.anthropic.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "llms/anthropic/chat/handler.py", rationale: "Anthropic vector API (verify support)"} - {id: llm.batches.openai.create.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Core batch create"} - {id: llm.batches.openai.retrieve.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Batch retrieve, id round-trip + status"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index 931085cbbc2..abef3266aa8 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -59,6 +59,7 @@ LlmRoute = Literal[ "ollama_chat", "openai", "sail", + "sagemaker", "together_ai", "vertex", "xiaomi_mimo", @@ -78,6 +79,7 @@ LlmCapability = Literal[ "multi_turn", "native_passthrough", "pdf_input", + "logprobs", "prompt_cache_1h", "prompt_cache_5m", "response_headers", @@ -85,6 +87,7 @@ LlmCapability = Literal[ "structured_output", "thinking", "thinking_with_tool_use", + "top_k", "tool_search", "tool_search_history", "tool_use", diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 902c161e425..fd5c0bae27b 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -213,6 +213,8 @@ OWNED_GATEWAY_OPT_IN_ENV: Final = "E2E_OWNED_GATEWAY" OTEL_V2_OPT_IN_ENV: Final = "E2E_OTEL_V2" OTEL_TLS_OPT_IN_ENV: Final = "E2E_OTEL_EXPORTER_ENDPOINT" SECRET_MANAGER_OPT_IN_ENV: Final = "E2E_SECRET_MANAGER" +SAGEMAKER_NOVA_OPT_IN_ENV: Final = "SAGEMAKER_NOVA_ENDPOINT" +SAGEMAKER_NOVA2_LITE_OPT_IN_ENV: Final = "SAGEMAKER_NOVA2_LITE_ENDPOINT" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) diff --git a/tests/e2e/llm_translation/test_bedrock_native_e2e.py b/tests/e2e/llm_translation/test_bedrock_native_e2e.py index a9b9e45858d..23ff6ac612e 100644 --- a/tests/e2e/llm_translation/test_bedrock_native_e2e.py +++ b/tests/e2e/llm_translation/test_bedrock_native_e2e.py @@ -7,22 +7,40 @@ missing messages and invalid model handling without crashing the proxy. from __future__ import annotations import pytest +from typing import Final + +from pydantic import BaseModel, ConfigDict + from e2e_config import unique_marker -from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from e2e_http import ( assert_client_error, require_successful_call, ) +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient -from pydantic import BaseModel +from sdk_clients import SdkClients pytestmark = pytest.mark.e2e BEDROCK_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" +class CalendarEvent(BaseModel): + model_config = ConfigDict(frozen=True) + + name: str + date: str + participants: tuple[str, ...] + + +class EventsList(BaseModel): + model_config = ConfigDict(frozen=True) + + events: tuple[CalendarEvent, ...] + + class ConverseContent(BaseModel): text: str @@ -100,6 +118,32 @@ def _default_invoke() -> InvokeBody: class TestBedrockNative: + @pytest.mark.provider_live + @pytest.mark.covers("llm.chat_completions.bedrock_converse.structured_output.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.BEDROCK,), + models=(BEDROCK_BACKEND,), + capabilities=(Capability.RESPONSE_SCHEMA,), + mode=Mode.NONSTREAM, + ) + ) + def test_bedrock_chat_completion_pydantic_response( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model, key = _register(proxy, resources) + response: Final = sdk.openai(key).chat.completions.parse( + model=model, + messages=[{"role": "user", "content": "Create one event for the Apollo 11 launch in July 1969."}], + response_format=EventsList, + ) + parsed: Final = response.choices[0].message.parsed + assert parsed is not None + assert parsed.events + assert all(event.name and event.date and event.participants for event in parsed.events) + @pytest.mark.covers("llm.bedrock_native.bedrock_converse.basic.nonstream.works") @meta( Subject( diff --git a/tests/e2e/llm_translation/test_cache_control.py b/tests/e2e/llm_translation/test_cache_control.py index 33c52efe37e..d3a8d33a386 100644 --- a/tests/e2e/llm_translation/test_cache_control.py +++ b/tests/e2e/llm_translation/test_cache_control.py @@ -40,7 +40,7 @@ from collections.abc import Callable from typing import Final import pytest -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from e2e_config import unique_marker from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta @@ -65,6 +65,8 @@ VERTEX_CACHE_REJECTION_MARKER: Final = "minimum token count to start explicit ca class CacheChatBody(BaseModel): + model_config = ConfigDict(frozen=True) + model: str messages: list[RichMessage] max_tokens: int = 64 @@ -309,6 +311,79 @@ class TestCacheControl: key = resources.key() _assert_cache_read_on_second_call(model, lambda prefix: _cache_chat(client, key, model, prefix)) + @pytest.mark.covers("llm.chat_completions.anthropic.prompt_cache_5m.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_MODEL,), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) + def test_anthropic_prompt_caching_reads_cache_with_version_header( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model: Final = f"e2e-anthropic-cache-version-{unique_marker()}" + model_id: Final = client.proxy.create_model( + model, + LiteLLMParamsBody( + model=ANTHROPIC_MODEL, + api_key="os.environ/ANTHROPIC_API_KEY", + extra_headers={"anthropic-version": "2023-06-01"}, + ), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key: Final = resources.key() + _assert_cache_read_on_second_call(model, lambda prefix: _cache_chat(client, key, model, prefix)) + + @pytest.mark.covers("llm.chat_completions.anthropic.prompt_cache_5m.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_MODEL,), + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) + def test_anthropic_prompt_caching_reports_creation_tokens( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model: Final = f"e2e-anthropic-cache-create-{unique_marker()}" + model_id: Final = client.proxy.create_model( + model, + LiteLLMParamsBody(model=ANTHROPIC_MODEL, api_key="os.environ/ANTHROPIC_API_KEY"), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key: Final = resources.key() + body: Final = CacheChatBody( + model=model, + messages=[ + RichMessage( + role="system", + content=[TextBlock(text=_cacheable_prefix(), cache_control=CacheControl(ttl="1h"))], + ), + RichMessage( + role="user", + content=[TextBlock(text="Reply with one word.", cache_control=CacheControl(ttl="5m"))], + ), + ], + ) + response: Final = unwrap( + client.proxy.transport.post( + "/chat/completions", + headers=client.proxy.transport.bearer(key), + json=body, + response_type=ChatResponse, + ) + ) + assert response.usage and response.usage.cache_creation_input_tokens, ( + f"{model}: a never-seen prefix marked 1h then 5m reported no cache-creation tokens ({response.usage})" + ) + @pytest.mark.covers( "llm.chat_completions.openai.prompt_cache_5m.nonstream.works", exercised_on=[], diff --git a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py index 7b59ffbf15f..5f5d2fee3f0 100644 --- a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py @@ -10,17 +10,20 @@ SDK refuses to build stay on the shared transport. from __future__ import annotations +import base64 import math -from typing import Final +from pathlib import Path +from typing import Final, Literal import pytest +from pydantic import BaseModel, ConfigDict + from e2e_config import provider_edge_base, unique_marker from e2e_http import assert_client_error -from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient -from pydantic import BaseModel from sdk_clients import NO_PROXY_CACHE, SdkClients, response_header pytestmark = pytest.mark.e2e @@ -28,6 +31,8 @@ pytestmark = pytest.mark.e2e OPENAI_EMBEDDING: Final = "openai/text-embedding-3-small" BEDROCK_TITAN_EMBEDDING: Final = "bedrock/amazon.titan-embed-text-v2:0" COHERE_EMBEDDING: Final = "cohere/embed-v4.0" +COHERE_IMAGE_EMBEDDING: Final = "cohere/embed-english-v3.0" +CAT_IMAGE: Final = Path(__file__).parent / "fixtures" / "cat.jpg" MISTRAL_EMBEDDING: Final = "mistral/mistral-embed" VERTEX_TEXT_EMBEDDING: Final = "vertex_ai/text-embedding-005" VERTEX_MULTIMODAL_EMBEDDING: Final = "vertex_ai/multimodalembedding@001" @@ -42,6 +47,29 @@ class _OptionalEmbeddingsBody(BaseModel): input: str | list[str] | None = None +class CohereImageTokenDetails(BaseModel): + model_config = ConfigDict(frozen=True) + + image_tokens: int | None = None + text_tokens: int | None = None + + +class CohereImageUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + prompt_tokens_details: CohereImageTokenDetails | None = None + + +class CohereImageEmbeddingResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + usage: CohereImageUsage | None = None + + +def _cat_image_data_uri() -> str: + return f"data:image/jpeg;base64,{base64.b64encode(CAT_IMAGE.read_bytes()).decode('ascii')}" + + def _cosine(left: list[float], right: list[float]) -> float: dot: Final = sum(a * b for a, b in zip(left, right, strict=True)) norms: Final = math.sqrt(sum(a * a for a in left)) * math.sqrt(sum(b * b for b in right)) @@ -156,6 +184,48 @@ class TestEmbeddingsEndpoint: LiteLLMParamsBody(model=COHERE_EMBEDDING, api_key="os.environ/COHERE_API_KEY"), ) + @pytest.mark.covers("llm.embeddings.cohere.vision.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.EMBEDDINGS, + providers=(Provider.COHERE,), + models=(COHERE_IMAGE_EMBEDDING,), + capabilities=(Capability.VISION,), + mode=Mode.NONSTREAM, + ) + ) + @pytest.mark.parametrize( + ("input_values", "token_type"), + ( + pytest.param((_cat_image_data_uri(),), "image_tokens", id="image"), + pytest.param((TOKENS_TEXT,), "text_tokens", id="text"), + ), + ) + def test_cohere_image_embeddings_report_image_tokens( + self, + proxy: ProxyClient, + resources: ResourceManager, + sdk: SdkClients, + input_values: tuple[str, ...], + token_type: Literal["image_tokens", "text_tokens"], + ) -> None: + model: Final = f"e2e-cohere-image-{unique_marker()}" + model_id: Final = proxy.create_model( + model, + LiteLLMParamsBody(model=COHERE_IMAGE_EMBEDDING, api_key="os.environ/COHERE_API_KEY"), + ) + resources.defer(lambda: proxy.delete_model(model_id)) + key: Final = resources.key() + embeddings: Final = sdk.openai(key).embeddings.create( + model=model, input=list(input_values), extra_body=NO_PROXY_CACHE + ) + response: Final = CohereImageEmbeddingResponse.model_validate(embeddings.model_dump()) + details: Final = response.usage.prompt_tokens_details if response.usage else None + assert details is not None, f"{model}: /embeddings returned no prompt_tokens_details: {response}" + reported: Final = details.image_tokens if token_type == "image_tokens" else details.text_tokens + assert reported, f"{model}: {token_type} missing or zero in {details}" + @pytest.mark.covers("llm.embeddings.vertex.basic.nonstream.works") @meta( Subject( diff --git a/tests/e2e/llm_translation/test_sagemaker_nova_e2e.py b/tests/e2e/llm_translation/test_sagemaker_nova_e2e.py new file mode 100644 index 00000000000..7960b5f7bf7 --- /dev/null +++ b/tests/e2e/llm_translation/test_sagemaker_nova_e2e.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import base64 +import os +from pathlib import Path +from typing import Final, Literal + +import pytest +from e2e_config import SAGEMAKER_NOVA2_LITE_OPT_IN_ENV, SAGEMAKER_NOVA_OPT_IN_ENV, unique_marker +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta +from lifecycle import ResourceManager +from models import LiteLLMParamsBody +from proxy_client import ProxyClient +from sdk_clients import SdkClients + +pytestmark = [pytest.mark.e2e, pytest.mark.provider_live] + +CAT_IMAGE: Final = Path(__file__).parent / "fixtures" / "cat.jpg" + + +def _deployment(proxy: ProxyClient, resources: ResourceManager, endpoint_env: str, label: str) -> tuple[str, str]: + endpoint: Final = os.environ[endpoint_env] + model_name: Final = f"sagemaker_nova/{endpoint}" + model: Final = f"e2e-{label}-{unique_marker()}" + model_id: Final = proxy.create_model(model, LiteLLMParamsBody(model=model_name)) + resources.defer(lambda: proxy.delete_model(model_id)) + return model, resources.key() + + +def _cat_image_data_uri() -> str: + return f"data:image/jpeg;base64,{base64.b64encode(CAT_IMAGE.read_bytes()).decode('ascii')}" + + +@pytest.mark.sagemaker_nova +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.SAGEMAKER,), + mode=Mode.NONSTREAM, + ) +) +class TestSagemakerNova: + @pytest.mark.covers("llm.chat_completions.sagemaker.vision.nonstream.works") + def test_multimodal_image_input_is_accepted( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model, key = _deployment(proxy, resources, SAGEMAKER_NOVA_OPT_IN_ENV, "nova-image") + response: Final = sdk.openai(key).chat.completions.create( + model=model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What animal is in this image? Answer in one word."}, + { + "type": "image_url", + "image_url": {"url": _cat_image_data_uri()}, + }, + ], + } + ], + max_tokens=64, + ) + answer: Final = response.choices[0].message.content or "" + assert "cat" in answer.lower(), f"{model}: the image did not reach the model, it answered {answer!r}" + + @pytest.mark.covers("llm.chat_completions.sagemaker.top_k.nonstream.works") + def test_nova_specific_top_k_parameter_is_accepted( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model, key = _deployment(proxy, resources, SAGEMAKER_NOVA_OPT_IN_ENV, "nova-top-k") + response: Final = sdk.openai(key).chat.completions.create( + model=model, + messages=[{"role": "user", "content": "Say hello."}], + max_tokens=32, + extra_body={"top_k": 40}, + ) + assert response.choices[0].message.content + + @pytest.mark.covers("llm.chat_completions.sagemaker.logprobs.nonstream.works") + def test_logprobs_returns_token_details( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model, key = _deployment(proxy, resources, SAGEMAKER_NOVA_OPT_IN_ENV, "nova-logprobs") + response: Final = sdk.openai(key).chat.completions.create( + model=model, + messages=[{"role": "user", "content": "Reply with the word hello."}], + max_tokens=8, + logprobs=True, + top_logprobs=3, + ) + logprobs: Final = response.choices[0].logprobs + assert logprobs and logprobs.content, f"{model}: no logprobs content in {response}" + assert len(logprobs.content[0].top_logprobs) == 3, f"{model}: top_logprobs=3 not honored: {logprobs.content[0]}" + + +@pytest.mark.sagemaker_nova2_lite +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.SAGEMAKER,), + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) +) +class TestSagemakerNova2Lite: + @pytest.mark.parametrize("reasoning_effort", ("low", "high")) + @pytest.mark.covers("llm.chat_completions.sagemaker.thinking.nonstream.works") + def test_reasoning_effort_is_accepted( + self, + reasoning_effort: Literal["low", "high"], + proxy: ProxyClient, + resources: ResourceManager, + sdk: SdkClients, + ) -> None: + model, key = _deployment(proxy, resources, SAGEMAKER_NOVA2_LITE_OPT_IN_ENV, "nova2-reasoning") + response: Final = sdk.openai(key).chat.completions.create( + model=model, + messages=[{"role": "user", "content": "What is 2 + 2?"}], + max_tokens=32, + reasoning_effort=reasoning_effort, + ) + assert response.choices[0].message.content diff --git a/tests/e2e/llm_translation/test_vertex_capabilities_e2e.py b/tests/e2e/llm_translation/test_vertex_capabilities_e2e.py new file mode 100644 index 00000000000..577d4a57f61 --- /dev/null +++ b/tests/e2e/llm_translation/test_vertex_capabilities_e2e.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +from typing import Final, TypedDict, cast + +import pytest +from typing_extensions import NotRequired, ReadOnly + +from e2e_config import unique_marker +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta +from lifecycle import ResourceManager +from models import LiteLLMParamsBody +from proxy_client import ProxyClient +from sdk_clients import SdkClients + +pytestmark = [pytest.mark.e2e, pytest.mark.provider_live] + +GEMINI_MODEL: Final = "vertex_ai/gemini-3-flash-preview" + + +class GoogleSearchTool(TypedDict): + googleSearch: ReadOnly[dict[str, str]] + + +class VertexGroundingBody(TypedDict): + tools: ReadOnly[tuple[GoogleSearchTool, ...]] + + +class GoogleMapsOptions(TypedDict): + enableWidget: ReadOnly[bool] + latitude: NotRequired[ReadOnly[float]] + longitude: NotRequired[ReadOnly[float]] + languageCode: NotRequired[ReadOnly[str]] + + +class GoogleMapsTool(TypedDict): + googleMaps: ReadOnly[GoogleMapsOptions] + + +class VertexMapsBody(TypedDict): + tools: ReadOnly[tuple[GoogleMapsTool, ...]] + + +def _register(proxy: ProxyClient, resources: ResourceManager, base: str, model: str, location: str) -> tuple[str, str]: + model_name: Final = f"{base}-{unique_marker()}" + model_id: Final = proxy.create_model( + model_name, + LiteLLMParamsBody( + model=model, + vertex_project="os.environ/VERTEXAI_PROJECT", + vertex_location=location, + vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", + ), + ) + resources.defer(lambda: proxy.delete_model(model_id)) + return model_name, resources.key() + + +class TestVertexCapabilities: + @pytest.mark.covers("llm.chat_completions.vertex.tool_use.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.VERTEX_AI,), + models=(GEMINI_MODEL,), + mode=Mode.NONSTREAM, + ) + ) + def test_google_maps_tool_accepts_widget_and_location( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model, key = _register(proxy, resources, "e2e-vertex-maps", GEMINI_MODEL, "global") + options: Final[tuple[GoogleMapsOptions, ...]] = ( + {"enableWidget": True}, + {"enableWidget": True, "latitude": 37.7749, "longitude": -122.4194, "languageCode": "en_US"}, + ) + client: Final = sdk.openai(key) + bodies: Final[tuple[VertexMapsBody, ...]] = tuple( + {"tools": ({"googleMaps": maps_options},)} for maps_options in options + ) + responses: Final = tuple( + client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": "What restaurants are nearby?"}], + extra_body=body, + ) + for body in bodies + ) + assert all(response.choices[0].message.content for response in responses) + + @pytest.mark.covers("llm.chat_completions.vertex.web_search.stream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.CHAT_COMPLETIONS, + providers=(Provider.VERTEX_AI,), + models=(GEMINI_MODEL,), + capabilities=(Capability.WEB_SEARCH,), + mode=Mode.STREAM, + ) + ) + def test_google_search_grounding_metadata_is_returned( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model, key = _register(proxy, resources, "e2e-vertex-grounding", GEMINI_MODEL, "global") + grounding_body: Final[VertexGroundingBody] = {"tools": ({"googleSearch": {}},)} + chunks: Final = sdk.openai(key).chat.completions.create( + model=model, + messages=[{"role": "user", "content": "What is today's weather in San Francisco?"}], + stream=True, + extra_body=grounding_body, + ) + metadata: Final = tuple( + cast(object | None, getattr(chunk, "vertex_ai_grounding_metadata", None)) for chunk in chunks + ) + assert any(value is not None for value in metadata) diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index dbd3ff47daa..ff81a593fe8 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -20,3 +20,5 @@ markers = otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set secret_manager: needs a proxy booted from gateway/secret_manager__ci_config.yml against that live secret manager; deselected unless E2E_SECRET_MANAGER names the backend (see secret_manager/secret_backends.py) + sagemaker_nova: needs a deployed SageMaker Nova endpoint the proxy's AWS credentials can invoke; deselected unless SAGEMAKER_NOVA_ENDPOINT names it + sagemaker_nova2_lite: needs a deployed SageMaker Nova 2 Lite endpoint the proxy's AWS credentials can invoke; deselected unless SAGEMAKER_NOVA2_LITE_ENDPOINT names it diff --git a/tests/integration/_support/upstream.py b/tests/integration/_support/upstream.py index 0c7dab90fcc..68d9207fa3c 100644 --- a/tests/integration/_support/upstream.py +++ b/tests/integration/_support/upstream.py @@ -17,7 +17,14 @@ from typing import Final, cast import httpx import uvicorn -from _fake_openai_endpoint_server import chat_completions, completions, embeddings, health, moderations +from _fake_openai_endpoint_server import ( + chat_completions, + completions, + embeddings, + health, + moderations, + triton_embeddings, +) from integration.cost_calculation.cost_tracking_case import ( BinaryResponse, EventStreamEvent, @@ -550,6 +557,7 @@ class Provider: Route("/v1/completions", completions, methods=["POST"]), Route("/v1/embeddings", embeddings, methods=["POST"]), Route("/v1/moderations", moderations, methods=["POST"]), + Route("/triton/embeddings", triton_embeddings, methods=["POST"]), Route("/vector_stores/{vector_store_id}/search", self.vector_store_search, methods=["POST"]), Route("/__interactions/{interaction_id}", self.interaction_state, methods=["PUT", "DELETE"]), Route("/v1beta/interactions/{interaction_id}:cancel", self.cancel_interaction, methods=["POST"]), diff --git a/tests/integration/providers/test_fake_openai_upstream_routes.py b/tests/integration/providers/test_fake_openai_upstream_routes.py new file mode 100644 index 00000000000..187adcd7d37 --- /dev/null +++ b/tests/integration/providers/test_fake_openai_upstream_routes.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from collections.abc import Iterator +from typing import Final + +import httpx +import pytest +from integration._support.client import JSON_OBJECT, Gateway, list_value, object_value +from openai.types.moderation_create_response import ModerationCreateResponse + + +@pytest.fixture +def upstream(gateway: Gateway) -> Iterator[httpx.Client]: + with httpx.Client(base_url=gateway.upstream_url, timeout=10, trust_env=False) as client: + yield client + + +def test_chat_completion_shape(upstream: httpx.Client) -> None: + response: Final = upstream.post( + "/v1/chat/completions", + json={"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]}, + ) + + assert response.status_code == 200, response.text + body: Final = JSON_OBJECT.validate_json(response.content) + message: Final = object_value(object_value(list_value(body["choices"])[0])["message"]) + assert message == { + "role": "assistant", + "content": "Hello! This is a mock response from the fake OpenAI endpoint.", + } + assert body["usage"] == { + "prompt_tokens": 20, + "completion_tokens": 20, + "total_tokens": 40, + } + + +def test_moderations_route_parses_as_an_openai_response(upstream: httpx.Client) -> None: + response: Final = upstream.post( + "/v1/moderations", + json={ + "input": ["I want to harm someone", "hello"], + "model": "omni-moderation-latest", + }, + ) + + assert response.status_code == 200, response.text + parsed: Final = ModerationCreateResponse.model_validate_json(response.content) + assert parsed.model == "omni-moderation-latest" + assert len(parsed.results) == 2 + assert parsed.results[0].categories.violence is False + + +def test_triton_embeddings_route(upstream: httpx.Client) -> None: + response: Final = upstream.post("/triton/embeddings", json={"inputs": []}) + + assert response.status_code == 200, response.text + output: Final = object_value(list_value(JSON_OBJECT.validate_json(response.content)["outputs"])[0]) + assert output["shape"] == [1, 2] + assert output["data"] == [0.1, 0.2] + + +def test_slow_model_blocks_past_client_timeout(upstream: httpx.Client) -> None: + with pytest.raises(httpx.TimeoutException): + upstream.post( + "/v1/chat/completions", + json={ + "model": "slow-endpoint", + "messages": [{"role": "user", "content": "hi"}], + }, + timeout=0.5, + ) diff --git a/tests/integration/providers/test_lm_studio_chat_wire.py b/tests/integration/providers/test_lm_studio_chat_wire.py new file mode 100644 index 00000000000..e2625674315 --- /dev/null +++ b/tests/integration/providers/test_lm_studio_chat_wire.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import json +from typing import Final + +from integration._support.client import JSON_OBJECT, Gateway, list_value, object_value +from integration._support.wire import Reply, Request, wire_server + +_BACKEND: Final = "typhoon2-quen2.5-7b-instruct" +_API_KEY: Final = "synthetic-lm-studio-key" +_PROMPT: Final = "What's the weather like in San Francisco?" +_ANSWER: Final = "Foggy and 14 degrees." + + +def test_lm_studio_completion(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/v1/chat/completions" + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + body: Final = JSON_OBJECT.validate_json(request.body) + assert body["model"] == _BACKEND + assert body["messages"] == [{"role": "user", "content": _PROMPT}] + return Reply( + body=json.dumps( + { + "id": "chatcmpl-lm-studio", + "object": "chat.completion", + "created": 1, + "model": _BACKEND, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": _ANSWER}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 12, + "completion_tokens": 6, + "total_tokens": 18, + }, + } + ).encode() + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"lm_studio/{_BACKEND}", api_base=f"{wire.url}/v1", api_key=_API_KEY) + response: Final = gateway.chat(model, text=_PROMPT) + received: Final = wire.drain() + + assert len(received) == 1 + message: Final = object_value(object_value(list_value(response["choices"])[0])["message"]) + assert message["content"] == _ANSWER + assert response["usage"] == { + "prompt_tokens": 12, + "completion_tokens": 6, + "total_tokens": 18, + } diff --git a/tests/integration/providers/test_openai_embedding_timeout_wire.py b/tests/integration/providers/test_openai_embedding_timeout_wire.py new file mode 100644 index 00000000000..7d2e8e818a6 --- /dev/null +++ b/tests/integration/providers/test_openai_embedding_timeout_wire.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import threading +from typing import Final + +from integration._support.client import JSON_OBJECT, Gateway, object_value +from integration._support.wire import Reply, Request, wire_server + +_API_KEY: Final = "synthetic-openai-key" + + +def test_openai_embedding_timeouts(gateway: Gateway) -> None: + release: Final = threading.Event() + + def respond(request: Request) -> Reply: + assert request.target == "/v1/embeddings" + assert JSON_OBJECT.validate_json(request.body)["input"] == ["good morning from litellm"] + release.wait(timeout=30) + return Reply(drop_connection=True) + + with wire_server(respond) as wire: + try: + with gateway.scenario() as scenario: + model: Final = scenario.model( + model="openai/slow-endpoint", + api_base=f"{wire.url}/v1", + api_key=_API_KEY, + timeout=0.5, + ) + response: Final = gateway.request( + "POST", + "/v1/embeddings", + {"model": model, "input": ["good morning from litellm"]}, + ) + finally: + release.set() + received: Final = wire.drain() + + assert response.status_code == 408, response.text + error: Final = object_value(JSON_OBJECT.validate_json(response.content)["error"]) + assert error["code"] == "408" + assert received + assert {request.headers["x-stainless-read-timeout"] for request in received} == {"0.5"} diff --git a/tests/integration/security/test_air_gapped_boot.py b/tests/integration/security/test_air_gapped_boot.py new file mode 100644 index 00000000000..731062bb11e --- /dev/null +++ b/tests/integration/security/test_air_gapped_boot.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import Final +from urllib.parse import urlsplit + +from integration._support.client import Gateway, string_value +from integration._support.forward_proxy import refusing_forward_proxy +from integration._support.process import owned_proxy + +_BUNDLED_COST_MAP: Final = Path(__file__).resolve().parents[3] / "litellm/model_prices_and_context_window_backup.json" + + +def _git_blob_id(body: bytes) -> str: + return hashlib.sha1(b"blob %d\0" % len(body) + body, usedforsecurity=False).hexdigest() + + +def test_proxy_becomes_ready_without_network(gateway: Gateway, tmp_path: Path) -> None: + with refusing_forward_proxy() as egress: + no_network: Final = { + "HTTPS_PROXY": egress.url, + "HTTP_PROXY": egress.url, + "ALL_PROXY": egress.url, + "NO_PROXY": "127.0.0.1,localhost", + } + with owned_proxy( + gateway, tmp_path, no_network, remove_environment=("LITELLM_LOCAL_MODEL_COST_MAP", "LITELLM_LICENSE") + ) as isolated: + readiness: Final = isolated.request("GET", "/health/readiness") + source: Final = isolated.get("/model/cost_map/source") + refused: Final = frozenset(egress.targets()) + + assert readiness.status_code == 200, readiness.text + assert source["source"] == "local" + assert source["is_env_forced"] is False + assert string_value(source["fallback_reason"]).startswith("Remote fetch failed") + assert source["source_revision"] == _git_blob_id(_BUNDLED_COST_MAP.read_bytes()) + assert refused == {f"{urlsplit(string_value(source['url'])).hostname}:443"} diff --git a/tests/integration/streaming/test_azure_streaming_function_calls.py b/tests/integration/streaming/test_azure_streaming_function_calls.py new file mode 100644 index 00000000000..a938e127e91 --- /dev/null +++ b/tests/integration/streaming/test_azure_streaming_function_calls.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import json +import os +import uuid +from collections.abc import Callable +from typing import Final + +from integration._support.client import JSON_OBJECT, Gateway, eventually, list_value, object_value +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue +from redis import Redis + +_TOOL: Final = { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a city", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, +} +_MESSAGES: Final = [{"role": "user", "content": "Weather in Boston?"}] + + +def _frame(stream_id: str, delta: JsonValue, finish_reason: str | None = None) -> bytes: + chunk: Final = { + "id": stream_id, + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4.1-mini", + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], + } + return f"data: {json.dumps(chunk)}\n\n".encode() + + +def _frames(stream_id: str) -> tuple[bytes, ...]: + return ( + _frame( + stream_id, + { + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "id": "call-weather", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": '{"location":"Bos', + }, + } + ], + }, + ), + _frame(stream_id, {"tool_calls": [{"index": 0, "function": {"arguments": 'ton","unit":"fahrenheit"}'}}]}), + _frame(stream_id, {}, finish_reason="tool_calls"), + b"data: [DONE]\n\n", + ) + + +def _responder(stream_id: str) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/openai/deployments/gpt-4.1-mini/chat/completions?api-version=2024-02-15-preview" + assert request.headers["api-key"] == "synthetic-azure-key" + body: Final = JSON_OBJECT.validate_json(request.body) + assert body["model"] == "gpt-4.1-mini" + assert body["messages"] == _MESSAGES + assert body["stream"] is True + assert body["tools"] == [_TOOL] + assert body["tool_choice"] == "auto" + return Reply(content_type="text/event-stream", chunks=_frames(stream_id)) + + return respond + + +def _stored(stream_id: str) -> bool: + with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as store: + return any( + stream_id.encode() in (store.get(key) or b"") + for key in store.scan_iter(count=1000) + if store.type(key) == b"string" + ) + + +def _tool_call_stream(gateway: Gateway, model: str) -> tuple[tuple[str, ...], str, tuple[str, ...]]: + with gateway.client.stream( + "POST", + "/v1/chat/completions", + json={ + "model": model, + "messages": _MESSAGES, + "tools": [_TOOL], + "tool_choice": "auto", + "stream": True, + }, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) as response: + assert response.status_code == 200, response.read().decode() + events: Final = tuple( + JSON_OBJECT.validate_json(line.removeprefix("data: ")) + for line in response.iter_lines() + if line.startswith("data: ") and line != "data: [DONE]" + ) + choices: Final = tuple(object_value(list_value(event["choices"])[0]) for event in events) + calls: Final = tuple( + object_value(call) + for choice in choices + for call in list_value(object_value(choice["delta"]).get("tool_calls") or []) + ) + functions: Final = tuple(object_value(call["function"]) for call in calls) + return ( + tuple(str(function["name"]) for function in functions if isinstance(function.get("name"), str)), + "".join(str(function["arguments"]) for function in functions if isinstance(function.get("arguments"), str)), + tuple(str(choice["finish_reason"]) for choice in choices if isinstance(choice.get("finish_reason"), str)), + ) + + +def test_azure_astreaming_and_function_calling(gateway: Gateway) -> None: + stream_id: Final = f"azure-stream-{uuid.uuid4().hex}" + with wire_server(_responder(stream_id)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="azure/gpt-4.1-mini", + api_base=wire.url, + api_key="synthetic-azure-key", + api_version="2024-02-15-preview", + ) + first: Final = _tool_call_stream(gateway, model) + eventually(lambda: _stored(stream_id), bool) + cached: Final = _tool_call_stream(gateway, model) + received: Final = wire.drain() + + expected: Final = ( + ("get_current_weather",), + '{"location":"Boston","unit":"fahrenheit"}', + ("tool_calls",), + ) + assert first == expected + assert cached == expected + assert len(received) == 1 diff --git a/tests/local_testing/model_cost.json b/tests/local_testing/model_cost.json deleted file mode 100644 index 8d6f6851e16..00000000000 --- a/tests/local_testing/model_cost.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "gpt-3.5-turbo": 7.7e-05 -} \ No newline at end of file diff --git a/tests/local_testing/stream_chunk_testdata.py b/tests/local_testing/stream_chunk_testdata.py deleted file mode 100644 index 6be9d1ebdff..00000000000 --- a/tests/local_testing/stream_chunk_testdata.py +++ /dev/null @@ -1,543 +0,0 @@ -from litellm.types.utils import ( - ChatCompletionDeltaToolCall, - Delta, - Function, - ModelResponse, - StreamingChoices, -) - -chunks = [ - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="To answer", - role="assistant", - function_call=None, - tool_calls=None, - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content=" your", role=None, function_call=None, tool_calls=None - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content=" question about", - role=None, - function_call=None, - tool_calls=None, - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content=" how", role=None, function_call=None, tool_calls=None - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content=" many rows are in the ", - role=None, - function_call=None, - tool_calls=None, - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="'users' table, I", - role=None, - function_call=None, - tool_calls=None, - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="'ll", role=None, function_call=None, tool_calls=None - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content=" need to", role=None, function_call=None, tool_calls=None - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content=" run", role=None, function_call=None, tool_calls=None - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content=" a SQL query.", - role=None, - function_call=None, - tool_calls=None, - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content=" Let", role=None, function_call=None, tool_calls=None - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content=" me", role=None, function_call=None, tool_calls=None - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content=" ", role=None, function_call=None, tool_calls=None - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="do that for", - role=None, - function_call=None, - tool_calls=None, - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content=" you.", role=None, function_call=None, tool_calls=None - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="", - role=None, - function_call=None, - tool_calls=[ - ChatCompletionDeltaToolCall( - id="toolu_01H3AjkLpRtGQrof13CBnWfK", - function=Function(arguments="", name="sql_query"), - type="function", - index=1, - ) - ], - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="", - role=None, - function_call=None, - tool_calls=[ - ChatCompletionDeltaToolCall( - id=None, - function=Function(arguments="", name=None), - type="function", - index=1, - ) - ], - ), - logprobs=None, - ) - ], - created=1722656356, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="", - role=None, - function_call=None, - tool_calls=[ - ChatCompletionDeltaToolCall( - id=None, - function=Function(arguments='{"', name=None), - type="function", - index=1, - ) - ], - ), - logprobs=None, - ) - ], - created=1722656357, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="", - role=None, - function_call=None, - tool_calls=[ - ChatCompletionDeltaToolCall( - id=None, - function=Function(arguments='query": ', name=None), - type="function", - index=1, - ) - ], - ), - logprobs=None, - ) - ], - created=1722656357, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="", - role=None, - function_call=None, - tool_calls=[ - ChatCompletionDeltaToolCall( - id=None, - function=Function(arguments='"SELECT C', name=None), - type="function", - index=1, - ) - ], - ), - logprobs=None, - ) - ], - created=1722656357, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="", - role=None, - function_call=None, - tool_calls=[ - ChatCompletionDeltaToolCall( - id=None, - function=Function(arguments="OUNT(*", name=None), - type="function", - index=1, - ) - ], - ), - logprobs=None, - ) - ], - created=1722656357, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="", - role=None, - function_call=None, - tool_calls=[ - ChatCompletionDeltaToolCall( - id=None, - function=Function(arguments=") ", name=None), - type="function", - index=1, - ) - ], - ), - logprobs=None, - ) - ], - created=1722656357, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="", - role=None, - function_call=None, - tool_calls=[ - ChatCompletionDeltaToolCall( - id=None, - function=Function(arguments="FROM use", name=None), - type="function", - index=1, - ) - ], - ), - logprobs=None, - ) - ], - created=1722656357, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="", - role=None, - function_call=None, - tool_calls=[ - ChatCompletionDeltaToolCall( - id=None, - function=Function(arguments='rs;"}', name=None), - type="function", - index=1, - ) - ], - ), - logprobs=None, - ) - ], - created=1722656357, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), - ModelResponse( - id="chatcmpl-634a6ad3-483a-44a1-8cdd-3befbeb4ac2f", - choices=[ - StreamingChoices( - finish_reason="tool_calls", - index=0, - delta=Delta( - content=None, role=None, function_call=None, tool_calls=None - ), - logprobs=None, - ) - ], - created=1722656357, - model="claude-3-5-sonnet-20240620", - object="chat.completion.chunk", - system_fingerprint=None, - ), -] diff --git a/tests/local_testing/test_acompletion.py b/tests/local_testing/test_acompletion.py deleted file mode 100644 index 6a06325e961..00000000000 --- a/tests/local_testing/test_acompletion.py +++ /dev/null @@ -1,25 +0,0 @@ -import pytest - - -@pytest.mark.asyncio -async def test_langfuse_double_logging(): - import litellm - - litellm.set_verbose = True - litellm.success_callback = ["langfuse"] - litellm.failure_callback = ["langfuse"] # logs errors to langfuse - - models = ["gpt-4o-mini", "claude-3-5-haiku-20241022"] - - messages = [ - {"role": "user", "content": "Hello, how are you?"}, - ] - - resp = await litellm.acompletion( - model=models[0], - messages=messages, - temperature=0.0, - fallbacks=models[1:], - # metadata={"generation_name": "test-gen", "project": "litellm-test"}, - ) - return resp diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py deleted file mode 100644 index 3152f446b7f..00000000000 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ /dev/null @@ -1,2337 +0,0 @@ -import os -import traceback - -from dotenv import load_dotenv - -load_dotenv() - - -import json -import tempfile -from unittest.mock import ANY, AsyncMock, MagicMock, patch - -import httpx -import pytest -from respx import MockRouter - -import litellm -from litellm import ( - acompletion, - completion, - embedding, - image_generation, -) -from litellm.llms.vertex_ai.gemini.transformation import ( - gemini_convert_messages_with_history, -) -from litellm.llms.vertex_ai.vertex_llm_base import VertexBase - -litellm.num_retries = 3 -litellm.cache = None -user_message = "Write a short poem about the sky" -messages = [{"content": user_message, "role": "user"}] - -VERTEX_MODELS_TO_NOT_TEST = [ - "medlm-medium", - "medlm-large", - "code-gecko", - "code-gecko@001", - "code-gecko@002", - "code-gecko@latest", - "codechat-bison@latest", - "code-bison@001", - "text-bison@001", - "gemini-1.5-pro", - "gemini-1.5-pro-preview-0215", - "gemini-pro-experimental", - "gemini-flash-experimental", - "gemini-2.5-flash-lite-exp-0827", - "gemini-2.0-pro-exp-02-05", - "gemini-pro-flash", - "gemini-2.5-flash-lite-exp-0827", - "gemini-2.0-flash-exp", - "gemini-2.0-flash-thinking-exp", - "gemini-2.0-flash-thinking-exp-01-21", - "gemini-2.0-flash-preview-image-generation", - "gemini-2.0-flash-live-preview-04-09", -] - - - - -def load_vertex_ai_credentials(): - # Define the path to the vertex_key.json file - print("loading vertex ai credentials") - filepath = os.path.dirname(os.path.abspath(__file__)) - vertex_key_path = filepath + "/vertex_key.json" - - # Read the existing content of the file or create an empty dictionary - try: - with open(vertex_key_path, "r") as file: - # Read the file content - print("Read vertexai file path") - content = file.read() - - # If the file is empty or not valid JSON, create an empty dictionary - if not content or not content.strip(): - service_account_key_data = {} - else: - # Attempt to load the existing JSON content - file.seek(0) - service_account_key_data = json.load(file) - except FileNotFoundError: - # If the file doesn't exist, create an empty dictionary - service_account_key_data = {} - - # Update the service_account_key_data with environment variables - private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "") - private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "") - private_key = private_key.replace("\\n", "\n") - service_account_key_data["private_key_id"] = private_key_id - service_account_key_data["private_key"] = private_key - - # Create a temporary file - with tempfile.NamedTemporaryFile(mode="w+", delete=False) as temp_file: - # Write the updated content to the temporary files - json.dump(service_account_key_data, temp_file, indent=2) - - # Export the temporary file as GOOGLE_APPLICATION_CREDENTIALS - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name) - - -# test_vertex_ai_anthropic_streaming() - - - - -# asyncio.run(test_vertex_ai_anthropic_async()) - - - - -# asyncio.run(test_vertex_ai_anthropic_async_streaming()) - - - - -# test_vertex_ai() - - - - -# test_vertex_ai_stream() - - -@pytest.mark.flaky(retries=3, delay=1) -@pytest.mark.asyncio -async def test_async_vertexai_streaming_response(): - import random - - litellm.turn_on_debug() - - load_vertex_ai_credentials() - test_models = ( - litellm.vertex_chat_models - | litellm.vertex_code_chat_models - | litellm.vertex_text_models - | litellm.vertex_code_text_models - ) - test_models = random.sample(list(test_models), 1) - test_models += list(litellm.vertex_language_models) # always test gemini-pro - test_models = ["gemini-3.5-flash"] - for model in test_models: - if model in VERTEX_MODELS_TO_NOT_TEST or ( - "gecko" in model - or "32k" in model - or "ultra" in model - or "002" in model - or "gemini-2.0-flash-thinking-exp" in model - or "gemini-2.0-pro-exp-02-05" in model - or "gemini-pro" in model - or "gemini-1.0-pro" in model - or "image-generation" in model - ): - # our account does not have access to this model - continue - try: - user_message = "Hello, how are you?" - messages = [{"content": user_message, "role": "user"}] - response = await acompletion( - model=model, - messages=messages, - temperature=0.7, - timeout=5, - stream=True, - vertex_location="global", - ) - print(f"response: {response}") - complete_response: str = "" - async for chunk in response: - print(f"chunk: {chunk}") - if chunk.choices[0].delta.content is not None: - complete_response += chunk.choices[0].delta.content - print(f"complete_response: {complete_response}") - except litellm.NotFoundError as e: - pass - except litellm.RateLimitError as e: - pass - except litellm.APIConnectionError: - pass - except litellm.Timeout as e: - pass - except litellm.InternalServerError as e: - pass - except Exception as e: - print(e) - pytest.fail(f"An exception occurred: {e}") - - - - - - -def vertex_httpx_grounding_post(*args, **kwargs): - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - mock_response.json.return_value = { - "candidates": [ - { - "content": { - "role": "model", - "parts": [ - { - "text": "Argentina won the FIFA World Cup 2022. Argentina defeated France 4-2 on penalties in the FIFA World Cup 2022 final tournament for the first time after 36 years and the third time overall." - } - ], - }, - "finishReason": "STOP", - "safetyRatings": [ - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.14940722, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.07477004, - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.15636235, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.015967654, - }, - { - "category": "HARM_CATEGORY_HARASSMENT", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.1943678, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.1284158, - }, - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.09384396, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.0726367, - }, - ], - "groundingMetadata": { - "webSearchQueries": ["who won the world cup 2022"], - "groundingAttributions": [ - { - "segment": {"endIndex": 38}, - "confidenceScore": 0.9919262, - "web": { - "uri": "https://www.careerpower.in/fifa-world-cup-winners-list.html", - "title": "FIFA World Cup Winners List from 1930 to 2022, Complete List - Career Power", - }, - }, - { - "segment": {"endIndex": 38}, - "confidenceScore": 0.9919262, - "web": { - "uri": "https://www.careerpower.in/fifa-world-cup-winners-list.html", - "title": "FIFA World Cup Winners List from 1930 to 2022, Complete List - Career Power", - }, - }, - { - "segment": {"endIndex": 38}, - "confidenceScore": 0.9919262, - "web": { - "uri": "https://www.britannica.com/sports/2022-FIFA-World-Cup", - "title": "2022 FIFA World Cup | Qatar, Controversy, Stadiums, Winner, & Final - Britannica", - }, - }, - { - "segment": {"endIndex": 38}, - "confidenceScore": 0.9919262, - "web": { - "uri": "https://en.wikipedia.org/wiki/2022_FIFA_World_Cup_final", - "title": "2022 FIFA World Cup final - Wikipedia", - }, - }, - { - "segment": {"endIndex": 38}, - "confidenceScore": 0.9919262, - "web": { - "uri": "https://www.transfermarkt.com/2022-world-cup/erfolge/pokalwettbewerb/WM22", - "title": "2022 World Cup - All winners - Transfermarkt", - }, - }, - { - "segment": {"startIndex": 39, "endIndex": 187}, - "confidenceScore": 0.9919262, - "web": { - "uri": "https://www.careerpower.in/fifa-world-cup-winners-list.html", - "title": "FIFA World Cup Winners List from 1930 to 2022, Complete List - Career Power", - }, - }, - { - "segment": {"startIndex": 39, "endIndex": 187}, - "confidenceScore": 0.9919262, - "web": { - "uri": "https://en.wikipedia.org/wiki/2022_FIFA_World_Cup_final", - "title": "2022 FIFA World Cup final - Wikipedia", - }, - }, - ], - "searchEntryPoint": { - "renderedContent": '\u003cstyle\u003e\n.container {\n align-items: center;\n border-radius: 8px;\n display: flex;\n font-family: Google Sans, Roboto, sans-serif;\n font-size: 14px;\n line-height: 20px;\n padding: 8px 12px;\n}\n.chip {\n display: inline-block;\n border: solid 1px;\n border-radius: 16px;\n min-width: 14px;\n padding: 5px 16px;\n text-align: center;\n user-select: none;\n margin: 0 8px;\n -webkit-tap-highlight-color: transparent;\n}\n.carousel {\n overflow: auto;\n scrollbar-width: none;\n white-space: nowrap;\n margin-right: -12px;\n}\n.headline {\n display: flex;\n margin-right: 4px;\n}\n.gradient-container {\n position: relative;\n}\n.gradient {\n position: absolute;\n transform: translate(3px, -9px);\n height: 36px;\n width: 9px;\n}\n@media (prefers-color-scheme: light) {\n .container {\n background-color: #fafafa;\n box-shadow: 0 0 0 1px #0000000f;\n }\n .headline-label {\n color: #1f1f1f;\n }\n .chip {\n background-color: #ffffff;\n border-color: #d2d2d2;\n color: #5e5e5e;\n text-decoration: none;\n }\n .chip:hover {\n background-color: #f2f2f2;\n }\n .chip:focus {\n background-color: #f2f2f2;\n }\n .chip:active {\n background-color: #d8d8d8;\n border-color: #b6b6b6;\n }\n .logo-dark {\n display: none;\n }\n .gradient {\n background: linear-gradient(90deg, #fafafa 15%, #fafafa00 100%);\n }\n}\n@media (prefers-color-scheme: dark) {\n .container {\n background-color: #1f1f1f;\n box-shadow: 0 0 0 1px #ffffff26;\n }\n .headline-label {\n color: #fff;\n }\n .chip {\n background-color: #2c2c2c;\n border-color: #3c4043;\n color: #fff;\n text-decoration: none;\n }\n .chip:hover {\n background-color: #353536;\n }\n .chip:focus {\n background-color: #353536;\n }\n .chip:active {\n background-color: #464849;\n border-color: #53575b;\n }\n .logo-light {\n display: none;\n }\n .gradient {\n background: linear-gradient(90deg, #1f1f1f 15%, #1f1f1f00 100%);\n }\n}\n\u003c/style\u003e\n\u003cdiv class="container"\u003e\n \u003cdiv class="headline"\u003e\n \u003csvg class="logo-light" width="18" height="18" viewBox="9 9 35 35" fill="none" xmlns="http://www.w3.org/2000/svg"\u003e\n \u003cpath fill-rule="evenodd" clip-rule="evenodd" d="M42.8622 27.0064C42.8622 25.7839 42.7525 24.6084 42.5487 23.4799H26.3109V30.1568H35.5897C35.1821 32.3041 33.9596 34.1222 32.1258 35.3448V39.6864H37.7213C40.9814 36.677 42.8622 32.2571 42.8622 27.0064V27.0064Z" fill="#4285F4"/\u003e\n \u003cpath fill-rule="evenodd" clip-rule="evenodd" d="M26.3109 43.8555C30.9659 43.8555 34.8687 42.3195 37.7213 39.6863L32.1258 35.3447C30.5898 36.3792 28.6306 37.0061 26.3109 37.0061C21.8282 37.0061 18.0195 33.9811 16.6559 29.906H10.9194V34.3573C13.7563 39.9841 19.5712 43.8555 26.3109 43.8555V43.8555Z" fill="#34A853"/\u003e\n \u003cpath fill-rule="evenodd" clip-rule="evenodd" d="M16.6559 29.8904C16.3111 28.8559 16.1074 27.7588 16.1074 26.6146C16.1074 25.4704 16.3111 24.3733 16.6559 23.3388V18.8875H10.9194C9.74388 21.2072 9.06992 23.8247 9.06992 26.6146C9.06992 29.4045 9.74388 32.022 10.9194 34.3417L15.3864 30.8621L16.6559 29.8904V29.8904Z" fill="#FBBC05"/\u003e\n \u003cpath fill-rule="evenodd" clip-rule="evenodd" d="M26.3109 16.2386C28.85 16.2386 31.107 17.1164 32.9095 18.8091L37.8466 13.8719C34.853 11.082 30.9659 9.3736 26.3109 9.3736C19.5712 9.3736 13.7563 13.245 10.9194 18.8875L16.6559 23.3388C18.0195 19.2636 21.8282 16.2386 26.3109 16.2386V16.2386Z" fill="#EA4335"/\u003e\n \u003c/svg\u003e\n \u003csvg class="logo-dark" width="18" height="18" viewBox="0 0 48 48" xmlns="http://www.w3.org/2000/svg"\u003e\n \u003ccircle cx="24" cy="23" fill="#FFF" r="22"/\u003e\n \u003cpath d="M33.76 34.26c2.75-2.56 4.49-6.37 4.49-11.26 0-.89-.08-1.84-.29-3H24.01v5.99h8.03c-.4 2.02-1.5 3.56-3.07 4.56v.75l3.91 2.97h.88z" fill="#4285F4"/\u003e\n \u003cpath d="M15.58 25.77A8.845 8.845 0 0 0 24 31.86c1.92 0 3.62-.46 4.97-1.31l4.79 3.71C31.14 36.7 27.65 38 24 38c-5.93 0-11.01-3.4-13.45-8.36l.17-1.01 4.06-2.85h.8z" fill="#34A853"/\u003e\n \u003cpath d="M15.59 20.21a8.864 8.864 0 0 0 0 5.58l-5.03 3.86c-.98-2-1.53-4.25-1.53-6.64 0-2.39.55-4.64 1.53-6.64l1-.22 3.81 2.98.22 1.08z" fill="#FBBC05"/\u003e\n \u003cpath d="M24 14.14c2.11 0 4.02.75 5.52 1.98l4.36-4.36C31.22 9.43 27.81 8 24 8c-5.93 0-11.01 3.4-13.45 8.36l5.03 3.85A8.86 8.86 0 0 1 24 14.14z" fill="#EA4335"/\u003e\n \u003c/svg\u003e\n \u003cdiv class="gradient-container"\u003e\u003cdiv class="gradient"\u003e\u003c/div\u003e\u003c/div\u003e\n \u003c/div\u003e\n \u003cdiv class="carousel"\u003e\n \u003ca class="chip" href="https://www.google.com/search?q=who+won+the+world+cup+2022&client=app-vertex-grounding&safesearch=active"\u003ewho won the world cup 2022\u003c/a\u003e\n \u003c/div\u003e\n\u003c/div\u003e\n' - }, - }, - } - ], - "usageMetadata": { - "promptTokenCount": 6, - "candidatesTokenCount": 48, - "totalTokenCount": 54, - }, - } - - return mock_response - - -@pytest.mark.parametrize("value_in_dict", [{}, {"disable_attribution": False}]) # -def test_gemini_pro_grounding(value_in_dict): - try: - load_vertex_ai_credentials() - litellm.set_verbose = True - - tools = [{"googleSearchRetrieval": value_in_dict}] - - litellm.set_verbose = True - - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - client = HTTPHandler() - - with patch.object( - client, "post", side_effect=vertex_httpx_grounding_post - ) as mock_call: - resp = litellm.completion( - model="vertex_ai_beta/gemini-1.0-pro-001", - messages=[{"role": "user", "content": "Who won the world cup?"}], - tools=tools, - client=client, - ) - - mock_call.assert_called_once() - - print(mock_call.call_args.kwargs["json"]["tools"][0]) - - assert ( - "googleSearchRetrieval" - in mock_call.call_args.kwargs["json"]["tools"][0] - ) - assert ( - mock_call.call_args.kwargs["json"]["tools"][0]["googleSearchRetrieval"] - == value_in_dict - ) - - assert "vertex_ai_grounding_metadata" in resp._hidden_params - assert isinstance(resp._hidden_params["vertex_ai_grounding_metadata"], list) - - except litellm.InternalServerError: - pass - except litellm.RateLimitError: - pass - - -from test_completion import response_format_tests - - -@pytest.mark.parametrize( - "model,region", - [ - ("vertex_ai/mistral-small-2503", "us-central1"), - ("vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas", "us-south1"), - ("vertex_ai/openai/gpt-oss-20b-maas", "us-central1"), - ], -) -@pytest.mark.parametrize( - "sync_mode", - [True, False], -) # -@pytest.mark.flaky(retries=3, delay=1) -@pytest.mark.asyncio -async def test_partner_models_httpx(model, region, sync_mode): - try: - load_vertex_ai_credentials() - litellm.set_verbose = True - - messages = [ - { - "role": "system", - "content": "Your name is Litellm Bot, you are a helpful assistant", - }, - # User asks for their name and weather in San Francisco - { - "role": "user", - "content": "Hello, what is your name and can you tell me the weather?", - }, - ] - - data = { - "model": model, - "messages": messages, - "timeout": 10, - "vertex_ai_location": region, - } - if sync_mode: - response = litellm.completion(**data) - else: - response = await litellm.acompletion(**data) - - response_format_tests(response=response) - - print(f"response: {response}") - - assert isinstance(response._hidden_params["response_cost"], float) - except litellm.RateLimitError as e: - print("RateLimitError", e) - pass - except litellm.Timeout as e: - print("Timeout", e) - pass - except litellm.InternalServerError as e: - print("InternalServerError", e) - pass - except litellm.APIConnectionError as e: - print("APIConnectionError", e) - pass - except litellm.ServiceUnavailableError as e: - print("ServiceUnavailableError", e) - pass - except Exception as e: - print("got generic exception", e) - if "429 Quota exceeded" in str(e): - pass - else: - pytest.fail("An unexpected exception occurred - {}".format(str(e))) - - -def vertex_httpx_mock_reject_prompt_post(*args, **kwargs): - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - mock_response.json.return_value = { - "promptFeedback": {"blockReason": "OTHER"}, - "usageMetadata": {"promptTokenCount": 6285, "totalTokenCount": 6285}, - } - - return mock_response - - -def vertex_httpx_mock_post(url, data=None, json=None, headers=None, **kwargs): - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - mock_response.json.return_value = { - "candidates": [ - { - "finishReason": "RECITATION", - "safetyRatings": [ - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.14965563, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.13660839, - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.16344544, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.10230471, - }, - { - "category": "HARM_CATEGORY_HARASSMENT", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.1979091, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.06052939, - }, - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.1765296, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.18417984, - }, - ], - "citationMetadata": { - "citations": [ - { - "startIndex": 251, - "endIndex": 380, - "uri": "https://chocolatecake2023.blogspot.com/2023/02/taste-deliciousness-of-perfectly-baked.html?m=1", - }, - { - "startIndex": 393, - "endIndex": 535, - "uri": "https://skinnymixes.co.uk/blogs/food-recipes/peanut-butter-cup-cookies", - }, - { - "startIndex": 439, - "endIndex": 581, - "uri": "https://mast-producing-trees.org/aldis-chocolate-chips-are-peanut-and-tree-nut-free/", - }, - { - "startIndex": 1117, - "endIndex": 1265, - "uri": "https://github.com/frdrck100/To_Do_Assignments", - }, - { - "startIndex": 1146, - "endIndex": 1288, - "uri": "https://skinnymixes.co.uk/blogs/food-recipes/peanut-butter-cup-cookies", - }, - { - "startIndex": 1166, - "endIndex": 1299, - "uri": "https://www.girlversusdough.com/brookies/", - }, - { - "startIndex": 1780, - "endIndex": 1909, - "uri": "https://chocolatecake2023.blogspot.com/2023/02/taste-deliciousness-of-perfectly-baked.html?m=1", - }, - { - "startIndex": 1834, - "endIndex": 1964, - "uri": "https://newsd.in/national-cream-cheese-brownie-day-2023-date-history-how-to-make-a-cream-cheese-brownie/", - }, - { - "startIndex": 1846, - "endIndex": 1989, - "uri": "https://github.com/frdrck100/To_Do_Assignments", - }, - { - "startIndex": 2121, - "endIndex": 2261, - "uri": "https://recipes.net/copycat/hardee/hardees-chocolate-chip-cookie-recipe/", - }, - { - "startIndex": 2505, - "endIndex": 2671, - "uri": "https://www.tfrecipes.com/Oranges%20with%20dried%20cherries/", - }, - { - "startIndex": 3390, - "endIndex": 3529, - "uri": "https://github.com/quantumcognition/Crud-palm", - }, - { - "startIndex": 3568, - "endIndex": 3724, - "uri": "https://recipes.net/dessert/cakes/ultimate-easy-gingerbread/", - }, - { - "startIndex": 3640, - "endIndex": 3770, - "uri": "https://recipes.net/dessert/cookies/soft-and-chewy-peanut-butter-cookies/", - }, - ] - }, - } - ], - "usageMetadata": {"promptTokenCount": 336, "totalTokenCount": 336}, - } - return mock_response - - -@pytest.mark.parametrize("provider", ["vertex_ai_beta"]) # "vertex_ai", -@pytest.mark.parametrize("content_filter_type", ["prompt", "response"]) # "vertex_ai", -@pytest.mark.asyncio -@pytest.mark.flaky(retries=3, delay=1) -async def test_gemini_pro_json_schema_httpx_content_policy_error( - provider, content_filter_type -): - load_vertex_ai_credentials() - litellm.set_verbose = True - messages = [ - { - "role": "user", - "content": """ - -List 5 popular cookie recipes. - -Using this JSON schema: -```json -{'$defs': {'Recipe': {'properties': {'recipe_name': {'examples': ['Chocolate Chip Cookies', 'Peanut Butter Cookies'], 'maxLength': 100, 'title': 'The recipe name', 'type': 'string'}, 'estimated_time': {'anyOf': [{'minimum': 0, 'type': 'integer'}, {'type': 'null'}], 'default': None, 'description': 'The estimated time to make the recipe in minutes', 'examples': [30, 45], 'title': 'The estimated time'}, 'ingredients': {'examples': [['flour', 'sugar', 'chocolate chips'], ['peanut butter', 'sugar', 'eggs']], 'items': {'type': 'string'}, 'maxItems': 10, 'title': 'The ingredients', 'type': 'array'}, 'instructions': {'examples': [['mix', 'bake'], ['mix', 'chill', 'bake']], 'items': {'type': 'string'}, 'maxItems': 10, 'title': 'The instructions', 'type': 'array'}}, 'required': ['recipe_name', 'ingredients', 'instructions'], 'title': 'Recipe', 'type': 'object'}}, 'properties': {'recipes': {'items': {'$ref': '#/$defs/Recipe'}, 'maxItems': 11, 'title': 'The recipes', 'type': 'array'}}, 'required': ['recipes'], 'title': 'MyRecipes', 'type': 'object'} -``` - """, - } - ] - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - client = HTTPHandler() - - if content_filter_type == "prompt": - _side_effect = vertex_httpx_mock_reject_prompt_post - else: - _side_effect = vertex_httpx_mock_post - - with patch.object(client, "post", side_effect=_side_effect) as mock_call: - response = completion( - model="vertex_ai_beta/gemini-2.5-flash-lite", - messages=messages, - response_format={"type": "json_object"}, - client=client, - logging_obj=ANY, - ) - - assert response.choices[0].finish_reason == "content_filter" - - mock_call.assert_called_once() - - -def vertex_httpx_mock_post_valid_response(*args, **kwargs): - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - mock_response.json.return_value = { - "candidates": [ - { - "content": { - "role": "model", - "parts": [ - { - "text": """{ - "recipes": [ - {"recipe_name": "Chocolate Chip Cookies"}, - {"recipe_name": "Oatmeal Raisin Cookies"}, - {"recipe_name": "Peanut Butter Cookies"}, - {"recipe_name": "Sugar Cookies"}, - {"recipe_name": "Snickerdoodles"} - ] - }""" - } - ], - }, - "finishReason": "STOP", - "safetyRatings": [ - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.09790669, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.11736965, - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.1261379, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.08601588, - }, - { - "category": "HARM_CATEGORY_HARASSMENT", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.083441176, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.0355444, - }, - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.071981624, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.08108212, - }, - ], - } - ], - "usageMetadata": { - "promptTokenCount": 60, - "candidatesTokenCount": 55, - "totalTokenCount": 115, - }, - } - return mock_response - - -def vertex_httpx_mock_post_valid_response_anthropic(*args, **kwargs): - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - mock_response.json.return_value = { - "id": "msg_vrtx_013Wki5RFQXAspL7rmxRFjZg", - "type": "message", - "role": "assistant", - "model": "claude-3-5-sonnet-20240620", - "content": [ - { - "type": "tool_use", - "id": "toolu_vrtx_01YMnYZrToPPfcmY2myP2gEB", - "name": "json_tool_call", - "input": { - "values": { - "recipes": [ - {"recipe_name": "Chocolate Chip Cookies"}, - {"recipe_name": "Oatmeal Raisin Cookies"}, - {"recipe_name": "Peanut Butter Cookies"}, - {"recipe_name": "Snickerdoodle Cookies"}, - {"recipe_name": "Sugar Cookies"}, - ] - } - }, - } - ], - "stop_reason": "tool_use", - "stop_sequence": None, - "usage": {"input_tokens": 368, "output_tokens": 118}, - } - - return mock_response - - -def vertex_httpx_mock_post_invalid_schema_response(*args, **kwargs): - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - mock_response.json.return_value = { - "candidates": [ - { - "content": { - "role": "model", - "parts": [ - {"text": '[{"recipe_world": "Chocolate Chip Cookies"}]\n'} - ], - }, - "finishReason": "STOP", - "safetyRatings": [ - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.09790669, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.11736965, - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.1261379, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.08601588, - }, - { - "category": "HARM_CATEGORY_HARASSMENT", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.083441176, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.0355444, - }, - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.071981624, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.08108212, - }, - ], - } - ], - "usageMetadata": { - "promptTokenCount": 60, - "candidatesTokenCount": 55, - "totalTokenCount": 115, - }, - } - return mock_response - - -def vertex_httpx_mock_post_invalid_schema_response_anthropic(*args, **kwargs): - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - mock_response.json.return_value = { - "id": "msg_vrtx_013Wki5RFQXAspL7rmxRFjZg", - "type": "message", - "role": "assistant", - "model": "claude-3-5-sonnet-20240620", - "content": [{"text": "Hi! My name is Claude.", "type": "text"}], - "stop_reason": "end_turn", - "stop_sequence": None, - "usage": {"input_tokens": 368, "output_tokens": 118}, - } - return mock_response - - - - -@pytest.mark.parametrize( - "model, vertex_location, supports_response_schema", - [ - ("vertex_ai_beta/gemini-2.0-flash-001", "us-central1", True), - ("vertex_ai_beta/gemini-2.5-flash-lite", "us-central1", True), - ("vertex_ai/claude-3-5-sonnet@20240620", "us-east5", False), - ], -) -@pytest.mark.parametrize("invalid_response", [True, False]) -@pytest.mark.parametrize("enforce_validation", [True, False]) -@pytest.mark.asyncio -async def test_gemini_pro_json_schema_args_sent_httpx( - model, - supports_response_schema, - vertex_location, - invalid_response, - enforce_validation, -): - load_vertex_ai_credentials() - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - litellm.set_verbose = True - messages = [{"role": "user", "content": "List 5 cookie recipes"}] - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - response_schema = { - "type": "object", - "properties": { - "recipes": { - "type": "array", - "items": { - "type": "object", - "properties": {"recipe_name": {"type": "string"}}, - "required": ["recipe_name"], - }, - } - }, - "required": ["recipes"], - "additionalProperties": False, - } - client = HTTPHandler() - httpx_response = MagicMock() - if invalid_response is True: - if "claude" in model: - httpx_response.side_effect = vertex_httpx_mock_post_invalid_schema_response_anthropic - else: - httpx_response.side_effect = vertex_httpx_mock_post_invalid_schema_response - else: - if "claude" in model: - httpx_response.side_effect = vertex_httpx_mock_post_valid_response_anthropic - else: - httpx_response.side_effect = vertex_httpx_mock_post_valid_response - resp = None - with patch.object(client, "post", new=httpx_response) as mock_call: - litellm.set_verbose = True - print(f"model entering completion: {model}") - try: - resp = completion( - model=model, - messages=messages, - response_format={ - "type": "json_object", - "response_schema": response_schema, - "enforce_validation": enforce_validation, - }, - vertex_location=vertex_location, - client=client, - ) - print("Received={}".format(resp)) - if invalid_response is True and enforce_validation is True: - pytest.fail("Expected this to fail") - except litellm.JSONSchemaValidationError as e: - if invalid_response is False: - pytest.fail("Expected this to pass. Got={}".format(e)) - mock_call.assert_called_once() - if "claude" not in model: - print(mock_call.call_args.kwargs) - print(mock_call.call_args.kwargs["json"]["generationConfig"]) - if supports_response_schema: - gen_config = mock_call.call_args.kwargs["json"]["generationConfig"] - assert ( - "response_schema" in gen_config - or "response_json_schema" in gen_config - ), f"Expected response_schema or response_json_schema in {gen_config}" - else: - gen_config = mock_call.call_args.kwargs["json"]["generationConfig"] - assert ( - "response_schema" not in gen_config - and "response_json_schema" not in gen_config - ) - assert ( - "Use this JSON schema:" - in mock_call.call_args.kwargs["json"]["contents"][0]["parts"][1]["text"] - ) - elif resp is not None: - assert resp.model == model.split("/")[1] - - -@pytest.mark.asyncio -async def test_anthropic_message_via_anthropic_messages(): - from unittest.mock import AsyncMock - - from litellm.llms.custom_httpx.llm_http_handler import AsyncHTTPHandler - - load_vertex_ai_credentials() - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - litellm.set_verbose = True - client = AsyncHTTPHandler() - - httpx_response = AsyncMock() - httpx_response.side_effect = vertex_httpx_mock_post_valid_response_anthropic - - call_1_kwargs = {} - call_2_kwargs = {} - with patch.object(client, "post", new=httpx_response) as mock_call: - messages = [{"role": "user", "content": "List 5 cookie recipes"}] - response = await litellm.anthropic_messages( - model="vertex_ai/claude-3-5-sonnet@20240620", - messages=messages, - max_tokens=100, - client=client, - ) - - print(f"response: {response}") - assert mock_call.call_count == 1 - call_1_kwargs = mock_call.call_args.kwargs - - with patch.object(client, "post", new=httpx_response) as mock_call: - response_2 = await litellm.acompletion( - model="vertex_ai/claude-3-5-sonnet@20240620", - messages=messages, - max_tokens=100, - client=client, - ) - print(f"response_2: {response_2}") - call_args = mock_call.call_args - print(f"call_args: {call_args}") - call_2_kwargs = mock_call.call_args.kwargs - call_2_kwargs["url"] = call_args[0][0] - - """ - Compare Call 1 and Call 2 - - Expect: - - url - - headers - - data / json - - to be the same, except for the Authorization header. - """ - print(f"call_1_kwargs: {call_1_kwargs}") - print(f"call_2_kwargs: {call_2_kwargs}") - assert ( - call_1_kwargs["url"] == call_2_kwargs["url"] - ), f"Expected url to be the same, but got {call_1_kwargs['url']} and Expected {call_2_kwargs['url']}" - assert "Authorization".lower() in [ - k.lower() for k in call_1_kwargs["headers"].keys() - ], f"Expected Authorization header to be present in call_1_kwargs, but got {call_1_kwargs['headers'].keys()}" - assert "content-type".lower() in [ - k.lower() for k in call_1_kwargs["headers"].keys() - ], f"Expected Content-Type header to be present in call_1_kwargs, but got {call_1_kwargs['headers'].keys()}" - - ## validate request body - print(f"call 1 kwargs keys: {call_1_kwargs.keys()}") - print(f"call_2_kwargs['json']: {type(call_2_kwargs['json'])}") - print(f"call_1_kwargs['data']: {type(call_1_kwargs['data'])}") - call_1_kwargs_data = json.loads(call_1_kwargs["data"]) - for k, v in call_2_kwargs["json"].items(): - assert ( - k in call_1_kwargs_data - ), f"Expected {k} to be present in call_1_kwargs['data'], but got {call_1_kwargs_data.keys()}" - - - - -@pytest.mark.parametrize( - "model, vertex_location, supports_response_schema", - [ - ("vertex_ai_beta/gemini-2.0-flash-001", "us-central1", True), - ("vertex_ai_beta/gemini-2.5-flash-lite", "us-central1", True), - ("vertex_ai/claude-3-5-sonnet@20240620", "us-east5", False), - ], -) -@pytest.mark.parametrize("invalid_response", [True, False]) -@pytest.mark.parametrize("enforce_validation", [True, False]) -@pytest.mark.asyncio -async def test_gemini_pro_json_schema_args_sent_httpx_openai_schema( - model, - supports_response_schema, - vertex_location, - invalid_response, - enforce_validation, -): - from typing import List - - if enforce_validation: - litellm.enable_json_schema_validation = True - from pydantic import BaseModel - - load_vertex_ai_credentials() - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - litellm.set_verbose = True - messages = [{"role": "user", "content": "List 5 cookie recipes"}] - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - class Recipe(BaseModel): - recipe_name: str - - class ResponseSchema(BaseModel): - recipes: List[Recipe] - - client = HTTPHandler() - httpx_response = MagicMock() - if invalid_response is True: - if "claude" in model: - httpx_response.side_effect = vertex_httpx_mock_post_invalid_schema_response_anthropic - else: - httpx_response.side_effect = vertex_httpx_mock_post_invalid_schema_response - else: - if "claude" in model: - httpx_response.side_effect = vertex_httpx_mock_post_valid_response_anthropic - else: - httpx_response.side_effect = vertex_httpx_mock_post_valid_response - with patch.object(client, "post", new=httpx_response) as mock_call: - print("SENDING CLIENT POST={}".format(client.post)) - try: - resp = completion( - model=model, - messages=messages, - response_format=ResponseSchema, - vertex_location=vertex_location, - client=client, - ) - print("Received={}".format(resp)) - if invalid_response is True and enforce_validation is True: - pytest.fail("Expected this to fail") - except litellm.JSONSchemaValidationError as e: - if invalid_response is False: - pytest.fail("Expected this to pass. Got={}".format(e)) - mock_call.assert_called_once() - if "claude" not in model: - print(mock_call.call_args.kwargs) - print(mock_call.call_args.kwargs["json"]["generationConfig"]) - if supports_response_schema: - gen_config = mock_call.call_args.kwargs["json"]["generationConfig"] - assert ( - "response_schema" in gen_config - or "response_json_schema" in gen_config - ), f"Expected response_schema or response_json_schema in {gen_config}" - assert ( - "response_mime_type" - in mock_call.call_args.kwargs["json"]["generationConfig"] - ) - assert ( - mock_call.call_args.kwargs["json"]["generationConfig"]["response_mime_type"] - == "application/json" - ) - else: - gen_config = mock_call.call_args.kwargs["json"]["generationConfig"] - assert ( - "response_schema" not in gen_config - and "response_json_schema" not in gen_config - ) - assert ( - "Use this JSON schema:" - in mock_call.call_args.kwargs["json"]["contents"][0]["parts"][1]["text"] - ) - - -@pytest.mark.parametrize( - "model", ["gemini-2.5-flash-lite", "claude-3-5-sonnet@20240620"] -) -@pytest.mark.asyncio -async def test_gemini_pro_httpx_custom_api_base(model): - load_vertex_ai_credentials() - litellm.set_verbose = True - messages = [ - { - "role": "user", - "content": "Hello world", - } - ] - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - client = HTTPHandler() - - with patch.object(client, "post", new=MagicMock()) as mock_call: - try: - response = completion( - model="vertex_ai/{}".format(model), - messages=messages, - response_format={"type": "json_object"}, - client=client, - api_base="my-custom-api-base", - extra_headers={"hello": "world"}, - ) - except Exception as e: - traceback.print_exc() - print("Receives error - {}".format(str(e))) - - mock_call.assert_called_once() - - print(f"mock_call.call_args: {mock_call.call_args}") - print(f"mock_call.call_args.kwargs: {mock_call.call_args.kwargs}") - if "url" in mock_call.call_args.kwargs: - assert ( - "my-custom-api-base:generateContent" - == mock_call.call_args.kwargs["url"] - ) - else: - assert "my-custom-api-base:rawPredict" == mock_call.call_args[0][0] - if "headers" in mock_call.call_args.kwargs: - assert "hello" in mock_call.call_args.kwargs["headers"] - - -# gemini_pro_function_calling() - - -# asyncio.run(gemini_pro_async_function_calling()) - - - - - - - - - - - - - - - - - - -@pytest.mark.parametrize( - ("route", "provider"), - [ - ("completion", "vertex_ai"), - ("embedding", "vertex_ai"), - ("image_generation", "gemini"), - ], - ids=[ - "completion-vertex_ai", - "embedding-vertex_ai", - "image_generation-gemini", - ], -) -def test_litellm_api_base(monkeypatch, route, provider): - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - client = HTTPHandler() - import litellm - - monkeypatch.setattr(litellm, "api_base", "https://litellm.com") - load_vertex_ai_credentials() - if route == "image_generation" and provider == "gemini": - pytest.skip("Gemini does not support image generation") - with patch.object(client, "post", new=MagicMock()) as mock_client: - try: - if route == "completion": - response = completion( - model=f"{provider}/gemini-2.0-flash-001", - messages=[{"role": "user", "content": "Hello, world!"}], - client=client, - ) - elif route == "embedding": - response = embedding( - model=f"{provider}/gemini-2.0-flash-001", - input=["Hello, world!"], - client=client, - ) - elif route == "image_generation": - response = image_generation( - model=f"{provider}/gemini-2.0-flash-001", - prompt="Hello, world!", - client=client, - ) - except Exception as e: - print(e) - mock_client.assert_called() - assert mock_client.call_args.kwargs["url"].startswith("https://litellm.com") - - -def test_prompt_factory(): - messages = [ - { - "role": "system", - "content": "Your name is Litellm Bot, you are a helpful assistant", - }, - # User asks for their name and weather in San Francisco - { - "role": "user", - "content": "Hello, what is your name and can you tell me the weather?", - }, - # Assistant replies with a tool call - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_123", - "type": "function", - "index": 0, - "function": { - "name": "get_weather", - "arguments": '{"location":"San Francisco, CA"}', - }, - } - ], - }, - # The result of the tool call is added to the history - { - "role": "tool", - "tool_call_id": "call_123", - "content": "27 degrees celsius and clear in San Francisco, CA", - }, - # Now the assistant can reply with the result of the tool call. - ] - - translated_messages = gemini_convert_messages_with_history(messages=messages) - - print(f"\n\ntranslated_messages: {translated_messages}\ntranslated_messages") - - - - -@pytest.mark.asyncio -async def test_completion_fine_tuned_model(): - load_vertex_ai_credentials() - mock_response = AsyncMock() - mock_response.headers = {} - mock_response.status_code = 200 - - def return_val(): - return { - "candidates": [ - { - "content": { - "role": "model", - "parts": [ - { - "text": "A canvas vast, a boundless blue,\nWhere clouds paint tales and winds imbue.\nThe sun descends in fiery hue,\nStars shimmer bright, a gentle few.\n\nThe moon ascends, a pearl of light,\nGuiding travelers through the night.\nThe sky embraces, holds all tight,\nA tapestry of wonder, bright." - } - ], - }, - "finishReason": "STOP", - "safetyRatings": [ - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "probability": "NEGLIGIBLE", - "probabilityScore": 0.028930664, - "severity": "HARM_SEVERITY_NEGLIGIBLE", - "severityScore": 0.041992188, - }, - # ... other safety ratings ... - ], - "avgLogprobs": -0.95772853367765187, - } - ], - "usageMetadata": { - "promptTokenCount": 7, - "candidatesTokenCount": 71, - "totalTokenCount": 78, - }, - } - - mock_response.json = return_val - - expected_payload = { - "contents": [ - {"role": "user", "parts": [{"text": "Write a short poem about the sky"}]} - ] - } - - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - return_value=mock_response, - ) as mock_post: - # Act: Call the litellm.completion function - response = await litellm.acompletion( - model="vertex_ai_beta/4965075652664360960", - messages=[{"role": "user", "content": "Write a short poem about the sky"}], - ) - - # Assert - mock_post.assert_called_once() - url, kwargs = mock_post.call_args - print("url = ", url) - - # this is the fine-tuned model endpoint - assert ( - url[0] - == "https://us-central1-aiplatform.googleapis.com/v1/projects/litellm-ci-cd/locations/us-central1/endpoints/4965075652664360960:generateContent" - ) - - print("call args = ", kwargs) - args_to_vertexai = kwargs["json"] - - print("args to vertex ai call:", args_to_vertexai) - - assert args_to_vertexai == expected_payload - assert response.choices[0].message.content.startswith("A canvas vast") - assert response.choices[0].finish_reason == "stop" - assert response.usage.total_tokens == 78 - - # Optional: Print for debugging - print("Arguments passed to Vertex AI:", args_to_vertexai) - print("Response:", response) - - -def mock_gemini_request(*args, **kwargs): - print(f"kwargs: {kwargs}") - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - if "cachedContents" in kwargs["url"]: - mock_response.json.return_value = { - "name": "cachedContents/4d2kd477o3pg", - "model": "models/gemini-2.5-flash-lite-001", - "createTime": "2024-08-26T22:31:16.147190Z", - "updateTime": "2024-08-26T22:31:16.147190Z", - "expireTime": "2024-08-26T22:36:15.548934784Z", - "displayName": "", - "usageMetadata": {"totalTokenCount": 323383}, - } - else: - mock_response.json.return_value = { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "Please provide me with the text of the legal agreement" - } - ], - "role": "model", - }, - "finishReason": "MAX_TOKENS", - "index": 0, - "safetyRatings": [ - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "probability": "NEGLIGIBLE", - }, - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "probability": "NEGLIGIBLE", - }, - { - "category": "HARM_CATEGORY_HARASSMENT", - "probability": "NEGLIGIBLE", - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "probability": "NEGLIGIBLE", - }, - ], - } - ], - "usageMetadata": { - "promptTokenCount": 40049, - "candidatesTokenCount": 10, - "totalTokenCount": 40059, - "cachedContentTokenCount": 40012, - }, - } - - return mock_response - - -def mock_gemini_list_request(*args, **kwargs): - from litellm.types.llms.vertex_ai import ( - CachedContent, - CachedContentListAllResponseBody, - ) - - print(f"kwargs: {kwargs}") - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - mock_response.json.return_value = CachedContentListAllResponseBody( - cachedContents=[CachedContent(name="test", displayName="test")] - ) - - return mock_response - - -from litellm._uuid import uuid - - -@pytest.mark.parametrize( - "sync_mode", - [True, False], -) -@pytest.mark.asyncio -async def test_gemini_context_caching_anthropic_format(sync_mode): - from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler - - litellm.set_verbose = True - gemini_context_caching_messages = [ - # System Message - { - "role": "system", - "content": [ - { - "type": "text", - "text": "Here is the full text of a complex legal agreement {}".format( - uuid.uuid4() - ) - * 4000, - "cache_control": {"type": "ephemeral"}, - } - ], - }, - # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache. - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What are the key terms and conditions in this agreement?", - "cache_control": {"type": "ephemeral"}, - } - ], - }, - { - "role": "assistant", - "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo", - }, - # The final turn is marked with cache-control, for continuing in followups. - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What are the key terms and conditions in this agreement?", - } - ], - }, - ] - if sync_mode: - client = HTTPHandler(concurrent_limit=1) - else: - client = AsyncHTTPHandler(concurrent_limit=1) - with patch.object(client, "post", side_effect=mock_gemini_request) as mock_client: - try: - if sync_mode: - response = litellm.completion( - model="gemini/gemini-2.5-flash-lite-001", - messages=gemini_context_caching_messages, - temperature=0.2, - max_tokens=10, - client=client, - ) - else: - response = await litellm.acompletion( - model="gemini/gemini-2.5-flash-lite-001", - messages=gemini_context_caching_messages, - temperature=0.2, - max_tokens=10, - client=client, - ) - - except Exception as e: - print(e) - - assert mock_client.call_count == 2 - - first_call_args = mock_client.call_args_list[0].kwargs - - print(f"first_call_args: {first_call_args}") - - assert "cachedContents" in first_call_args["url"] - - # assert "cache_read_input_tokens" in response.usage - # assert "cache_creation_input_tokens" in response.usage - - # # Assert either a cache entry was created or cache was read - changes depending on the anthropic api ttl - # assert (response.usage.cache_read_input_tokens > 0) or ( - # response.usage.cache_creation_input_tokens > 0 - # ) - - - - -@pytest.mark.asyncio -async def test_partner_models_httpx_ai21(): - litellm.set_verbose = True - model = "vertex_ai/jamba-1.5-mini@001" - - messages = [ - { - "role": "system", - "content": "Your name is Litellm Bot, you are a helpful assistant", - }, - { - "role": "user", - "content": "Hello, can you tell me the weather in San Francisco?", - }, - ] - - tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - } - }, - "required": ["location"], - }, - }, - } - ] - - data = { - "model": model, - "messages": messages, - "tools": tools, - "top_p": 0.5, - } - - mock_response = AsyncMock() - - def return_val(): - return { - "id": "chat-3d11cf95eb224966937b216d9494fe73", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": " Sure, let me check that for you.", - "tool_calls": [ - { - "id": "b5cef16b-5946-4937-b9d5-beeaea871e77", - "type": "function", - "function": { - "name": "get_weather", - "arguments": '{"location": "San Francisco"}', - }, - } - ], - }, - "finish_reason": "stop", - } - ], - "usage": { - "prompt_tokens": 158, - "completion_tokens": 36, - "total_tokens": 194, - }, - "meta": {"requestDurationMillis": 501}, - "model": "jamba-1.5-mini@001", - } - - mock_response.json = return_val - mock_response.status_code = 200 - - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - return_value=mock_response, - ) as mock_post: - response = await litellm.acompletion(**data) - - # Assert - mock_post.assert_called_once() - url, kwargs = mock_post.call_args - print("url = ", url) - print("call args = ", kwargs) - - print(kwargs["data"]) - - assert ( - url[0] - == "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/litellm-ci-cd/locations/us-central1/publishers/ai21/models/jamba-1.5-mini@001:rawPredict" - ) - - # json loads kwargs - kwargs["data"] = json.loads(kwargs["data"]) - - assert kwargs["data"] == { - "model": "jamba-1.5-mini@001", - "messages": [ - { - "role": "system", - "content": "Your name is Litellm Bot, you are a helpful assistant", - }, - { - "role": "user", - "content": "Hello, can you tell me the weather in San Francisco?", - }, - ], - "top_p": 0.5, - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - } - }, - "required": ["location"], - }, - }, - } - ], - "stream": False, - } - - assert response.id == "chat-3d11cf95eb224966937b216d9494fe73" - assert len(response.choices) == 1 - assert ( - response.choices[0].message.content == " Sure, let me check that for you." - ) - assert response.choices[0].message.tool_calls[0].function.name == "get_weather" - assert ( - response.choices[0].message.tool_calls[0].function.arguments - == '{"location": "San Francisco"}' - ) - assert response.usage.prompt_tokens == 158 - assert response.usage.completion_tokens == 36 - assert response.usage.total_tokens == 194 - - print(f"response: {response}") - - - - - - -@pytest.mark.parametrize( - "base_model, metadata", - [ - (None, {"model_info": {"base_model": "vertex_ai/gemini-1.5-pro"}}), - ("vertex_ai/gemini-1.5-pro", None), - ], -) -def test_gemini_finetuned_endpoint(base_model, metadata): - litellm.set_verbose = True - load_vertex_ai_credentials() - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - # Set up the messages - messages = [ - {"role": "system", "content": """Use search for most queries."""}, - {"role": "user", "content": """search for weather in boston (use `search`)"""}, - ] - - client = HTTPHandler(concurrent_limit=1) - - with patch.object(client, "post", new=MagicMock()) as mock_client: - try: - response = completion( - model="vertex_ai/4965075652664360960", - messages=messages, - tool_choice="auto", - client=client, - metadata=metadata, - base_model=base_model, - ) - except Exception as e: - print(e) - - print(mock_client.call_args.kwargs) - - mock_client.assert_called() - assert mock_client.call_args.kwargs["url"].endswith( - "endpoints/4965075652664360960:generateContent" - ) - - - - -@pytest.mark.asyncio -@pytest.mark.respx -async def test_vertexai_embedding_finetuned(respx_mock: MockRouter): - """ - Tests that: - - Request URL and body are correctly formatted for Vertex AI embeddings - - Response is properly parsed into litellm's embedding response format - """ - load_vertex_ai_credentials() - litellm.set_verbose = True - litellm.disable_aiohttp_transport = ( - True # since this uses respx, we need to set use_aiohttp_transport to False - ) - - # Test input - input_text = ["good morning from litellm", "this is another item"] - - # Expected request/response - expected_url = "https://us-central1-aiplatform.googleapis.com/v1/projects/633608382793/locations/us-central1/endpoints/1004708436694269952:predict" - expected_request = { - "instances": [ - {"inputs": "good morning from litellm"}, - {"inputs": "this is another item"}, - ], - "parameters": {}, - } - - mock_response = { - "predictions": [ - [[-0.000431762, -0.04416759, -0.03443353]], # Truncated embedding vector - [[-0.000431762, -0.04416759, -0.03443353]], # Truncated embedding vector - ], - "deployedModelId": "2275167734310371328", - "model": "projects/633608382793/locations/us-central1/models/snowflake-arctic-embed-m-long-1731622468876", - "modelDisplayName": "snowflake-arctic-embed-m-long-1731622468876", - "modelVersionId": "1", - } - - # Setup mock request - mock_request = respx_mock.post(expected_url).mock( - return_value=httpx.Response(200, json=mock_response) - ) - - # Make request - response = await litellm.aembedding( - vertex_project="633608382793", - model="vertex_ai/1004708436694269952", - input=input_text, - ) - - # Assert request was made correctly - assert mock_request.called - request_body = json.loads(mock_request.calls[0].request.content) - print("\n\nrequest_body", request_body) - print("\n\nexpected_request", expected_request) - assert request_body == expected_request - - # Assert response structure - assert response is not None - assert hasattr(response, "data") - assert len(response.data) == len(input_text) - - # Assert embedding structure - for embedding in response.data: - assert "embedding" in embedding - assert isinstance(embedding["embedding"], list) - assert len(embedding["embedding"]) > 0 - assert all(isinstance(x, float) for x in embedding["embedding"]) - - -@pytest.mark.parametrize("max_retries", [None, 3]) -@pytest.mark.asyncio -@pytest.mark.respx -async def test_vertexai_model_garden_model_completion( - respx_mock: MockRouter, max_retries -): - """ - Relevant issue: https://github.com/BerriAI/litellm/issues/6480 - - Using OpenAI compatible models from Vertex Model Garden - """ - litellm.disable_aiohttp_transport = ( - True # since this uses respx, we need to set use_aiohttp_transport to False - ) - litellm.module_level_aclient = httpx.AsyncClient() - load_vertex_ai_credentials() - litellm.set_verbose = True - - # Test input - messages = [ - { - "role": "system", - "content": "Your name is Litellm Bot, you are a helpful assistant", - }, - { - "role": "user", - "content": "Hello, what is your name and can you tell me the weather?", - }, - ] - - # Expected request/response - expected_url = "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/633608382793/locations/us-central1/endpoints/5464397967697903616/chat/completions" - expected_request = {"model": "", "messages": messages, "stream": False} - - mock_response = { - "id": "chat-09940d4e99e3488aa52a6f5e2ecf35b1", - "object": "chat.completion", - "created": 1731702782, - "model": "meta-llama/Llama-3.1-8B-Instruct", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Hello, my name is Litellm Bot. I'm a helpful assistant here to provide information and answer your questions.\n\nTo check the weather for you, I'll need to know your location. Could you please provide me with your city or zip code? That way, I can give you the most accurate and up-to-date weather information.\n\nIf you don't have your location handy, I can also suggest some popular weather websites or apps that you can use to check the weather for your area.\n\nLet me know how I can assist you!", - "tool_calls": [], - }, - "logprobs": None, - "finish_reason": "stop", - "stop_reason": None, - } - ], - "usage": {"prompt_tokens": 63, "total_tokens": 172, "completion_tokens": 109}, - "prompt_logprobs": None, - } - - # Setup mock request - mock_request = respx_mock.post(expected_url).mock( - return_value=httpx.Response(200, json=mock_response) - ) - - # Make request - response = await litellm.acompletion( - model="vertex_ai/openai/5464397967697903616", - messages=messages, - vertex_project="633608382793", - vertex_location="us-central1", - max_retries=max_retries, - ) - - # Assert request was made correctly - assert mock_request.called - request_body = json.loads(mock_request.calls[0].request.content) - assert request_body == expected_request - - # Assert response structure - assert response.id == "chat-09940d4e99e3488aa52a6f5e2ecf35b1" - assert response.created == 1731702782 - assert response.model == "vertex_ai/meta-llama/Llama-3.1-8B-Instruct" - assert len(response.choices) == 1 - assert response.choices[0].message.role == "assistant" - assert response.choices[0].message.content.startswith( - "Hello, my name is Litellm Bot" - ) - assert response.choices[0].finish_reason == "stop" - assert response.usage.completion_tokens == 109 - assert response.usage.prompt_tokens == 63 - assert response.usage.total_tokens == 172 - - -def vertex_ai_anthropic_thinking_mock_response(*args, **kwargs): - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - mock_response.json.return_value = { - "id": "msg_vrtx_011pL6Np3MKxXL3R8theMRJW", - "type": "message", - "role": "assistant", - "model": "claude-4-sonnet-20250514", - "content": [ - { - "type": "thinking", - "thinking": 'This is a very simple and common greeting in programming and computing. "Hello, world!" is often the first program people write when learning a new programming language, where they create a program that outputs this phrase.\n\nI should respond in a friendly way and acknowledge this greeting. I can keep it simple and welcoming.', - "signature": "EugBCkYQAhgCIkAqCkezmsp8DG9Jjoc/CD7yXavPXVvP4TAuwjc/ZgHRIgroz5FzAYxic3CnNiW5w2fx/4+1f4ZYVxWJVLmrEA46EgwFsxbpN2jxMxjIzy0aDIAbMy9rW6B5lGVETCIw4r2UW0A7m5Df991SMSMPvHU9VdL8p9S/F2wajLnLVpl5tH89csm4NqnMpxnou61yKlCLldFGIto1Kvit5W1jqn2gx2dGIOyR4YaJ0c8AIFfQa5TIXf+EChVDzhPKLWZ8D/Q3gCGxBx+m/4dLI8HMZA8Ob3iCMI23eBKmh62FCWJGuA==", - }, - { - "type": "text", - "text": "Hi there! 👋 \n\nIt's nice to meet you! \"Hello, world!\" is such a classic phrase in computing - it's often the first output from someone's very first program.\n\nHow are you doing today? Is there something specific I can help you with?", - }, - ], - "stop_reason": "end_turn", - "stop_sequence": None, - "usage": { - "input_tokens": 39, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "output_tokens": 134, - }, - } - - return mock_response - - -def test_vertex_anthropic_completion(): - from litellm import completion - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - client = HTTPHandler() - - load_vertex_ai_credentials() - - with patch.object( - client, "post", side_effect=vertex_ai_anthropic_thinking_mock_response - ): - response = completion( - model="vertex_ai/claude-sonnet-4-6@default", - messages=[{"role": "user", "content": "Hello, world!"}], - vertex_ai_location="us-east5", - vertex_ai_project="test-project", - thinking={"type": "enabled", "budget_tokens": 1024}, - client=client, - ) - print(response) - assert response.model == "claude-sonnet-4-6@default" - assert response._hidden_params["response_cost"] is not None - assert response._hidden_params["response_cost"] > 0 - - assert response.choices[0].message.reasoning_content is not None - assert isinstance(response.choices[0].message.reasoning_content, str) - assert response.choices[0].message.thinking_blocks is not None - assert isinstance(response.choices[0].message.thinking_blocks, list) - assert len(response.choices[0].message.thinking_blocks) > 0 - - -def test_signed_s3_url_with_format(): - from litellm import completion - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - client = HTTPHandler() - - load_vertex_ai_credentials() - - args = { - "model": "vertex_ai/gemini-2.0-flash-001", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "image_url", - "image_url": { - "url": "https://litellm-logo-aws-marketplace.s3.us-west-2.amazonaws.com/berriai-logo-github.png?response-content-disposition=inline&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Security-Token=IQoJb3JpZ2luX2VjENj%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaCXVzLXdlc3QtMiJGMEQCIHlAy6QneghdEo4Dp4rw%2BHhdInKX4MU3T0hZT1qV3AD%2FAiBGY%2FtfxmBJkj%2BK6%2FxAgek6L3tpOcq6su1mBrj87El%2FCirLAwghEAEaDDg4ODYwMjIyMzQyOCIMzds7lsxAFHHCRHmkKqgDgnsJBaEmmwXBWqzyMMe3BUKsCqfvrYupFGxBREP%2BaEz%2ByLSKiTM3xWzaRz6vrP9T4HSJ97B9wQ3dhUBT22XzdOFsaq49wZapwy9hoPNrMyZ77DIa0MlEbg0uudGOaMAw4NbVEqoERQuZmIMMbNHCeoJsZxKCttRZlTDzU%2FeNNy96ltb%2FuIkX5b3OOYdUaKj%2FUjmPz%2FEufY%2Bn%2FFHawunSYXJwL4pYuBF1IKRtPjqamaYscH%2FrzD7fubGUMqk6hvyGEo%2BLqnVyruQEmVFqAnXyWlpHGqeWazEC7xcsC2lhLO%2FKUouyVML%2FxyYtL4CuKp52qtLWWauAFGnyBZnCHtSL58KLaMTSh7inhoFFIKDN2hymrJ4D9%2Bxv%2FMOzefH5X%2B0pcdJUwyxcwgL3myggRmIYq1L6IL4I%2F54BIU%2FMctJcRXQ8NhQNP2PsaCsXYHHVMXRZxps9v8t9Ciorb0PAaLr0DIGVgEqejSjwbzNTctQf59Rj0GhZ0A6A3nFaq3nL4UvO51aPP6aelN6RnLwHh8fF80iPWII7Oj9PWn9bkON%2F7%2B5k42oPFR0KDTD0yaO%2BBjrlAouRvkyHZnCuLuJdEeqc8%2Fwm4W8SbMiYDzIEPPe2wFR2sH4%2FDlnJRqia9Or00d4N%2BOefBkPv%2Bcdt68r%2FwjeWOrulczzLGjJE%2FGw1Lb9dtGtmupGm2XKOW3geJwXkk1qcr7u5zwy6DNamLJbitB026JFKorRnPajhe5axEDv%2BRu6l1f0eailIrCwZ2iytA94Ni8LTha2GbZvX7fFHcmtyNlgJPpMcELdkOEGTCNBldGck5MFHG27xrVrlR%2F7HZIkKYlImNmsOIjuK7acDiangvVdB6GlmVbzNUKtJ7YJhS2ivwvdDIf8XuaFAkhjRNpewDl0GzPvojK%2BDTizZydyJL%2B20pVkSXptyPwrrHEeiOFWwhszW2iTZij4rlRAoZW6NEdfkWsXrGMbxJTZa3E5URejJbg%2B4QgGtjLrgJhRC1pJGP02GX7VMxVWZzomfC2Hn7WaF44wgcuqjE4HGJfpA2ZLBxde52g%3D%3D&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ASIA45ZGR4NCKIUOODV3%2F20250305%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20250305T235823Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=71a900a9467eaf3811553500aaf509a10a9e743a8133cfb6a78dcbcbc6da4a05", - "format": "image/jpeg", - }, - }, - {"type": "text", "text": "Describe this image"}, - ], - } - ], - } - with patch.object(client, "post", new=MagicMock()) as mock_client: - try: - response = completion(**args, client=client) - print(response) - except Exception as e: - print(e) - - print(mock_client.call_args.kwargs) - - mock_client.assert_called() - - print(mock_client.call_args.kwargs) - - json_str = json.dumps(mock_client.call_args.kwargs["json"]) - assert "image/jpeg" in json_str - assert "image/png" not in json_str - - -def test_gemini_fine_tuned_model_request_consistency(): - """ - Assert the same transformation is applied to Fine tuned gemini 2.0 flash and gemini 2.0 flash - - - Request 1: Fine tuned: vertex_ai/gemini/ft-uuid - - Request 2: vertex_ai/gemini-2.5-flash - """ - litellm.set_verbose = True - load_vertex_ai_credentials() - from unittest.mock import MagicMock, patch - - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - # Set up the messages - messages = [ - { - "role": "system", - "content": "Your name is Litellm Bot, you are a helpful assistant", - }, - { - "role": "user", - "content": "Hello, what is your name and can you tell me the weather?", - }, - ] - - # Define tools - tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - } - }, - "required": ["location"], - }, - }, - } - ] - - client = HTTPHandler(concurrent_limit=1) - - # First request - with patch.object(client, "post", new=MagicMock()) as mock_post_1: - try: - response_1 = completion( - model="vertex_ai/gemini/ft-uuid", - messages=messages, - tools=tools, - tool_choice="auto", - client=client, - ) - - except Exception as e: - print(e) - - # Store the request body from the first call - first_request_body = mock_post_1.call_args.kwargs["json"] - print("first_request_body", first_request_body) - - # Validate correct `model` is added to the request to Vertex AI - print("final URL=", mock_post_1.call_args.kwargs["url"]) - # Validate the request url - assert ( - "publishers/google/models/ft-uuid:generateContent" - in mock_post_1.call_args.kwargs["url"] - ) - - # Second request - with patch.object(client, "post", new=MagicMock()) as mock_post_2: - try: - response_2 = completion( - model="vertex_ai/gemini-2.5-flash", - messages=messages, - tools=tools, - tool_choice="auto", - client=client, - ) - except Exception as e: - print(e) - - # Store the request body from the second call - second_request_body = mock_post_2.call_args.kwargs["json"] - print("second_request_body", second_request_body) - - # Get the diff between the two request bodies - # Convert dictionaries to formatted JSON strings - import json - - first_json = json.dumps(first_request_body, indent=2).splitlines() - second_json = json.dumps(second_request_body, indent=2).splitlines() - # Assert there is no difference between the request bodies - assert first_json == second_json, "Request bodies should be identical" - - - - - - - - -def test_vertex_ai_llama_tool_calling(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - load_vertex_ai_credentials() - litellm.turn_on_debug() - args = { - "model": "vertex_ai/meta/llama-4-maverick-17b-128e-instruct-maas", - "messages": [ - {"role": "user", "content": "What is the weather in Boston, MA today?"} - ], - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. Bogotá, Colombia", - } - }, - "required": ["location"], - "additionalProperties": False, - }, - }, - } - ], - "vertex_location": "us-east5", - } - try: - response = completion(**args) - except litellm.RateLimitError: - pytest.skip("Rate limit error") - except litellm.NotFoundError: - pytest.skip("Model not found / resource unavailable") - print(response) - - assert response.choices[0].message.tool_calls is not None - assert response.choices[0].finish_reason == "tool_calls" - assert response._hidden_params["response_cost"] > 0 - - -def test_gemini_nullable_object_tool_schema_httpx(): - """ - Ensure nullable object tool params preserve nested properties in Vertex schema conversion. - """ - load_vertex_ai_credentials() - litellm.turn_on_debug() - - tools = [ - { - "type": "function", - "strict": True, - "function": { - "name": "create_support_ticket", - "description": "Create a paid user support ticket", - "parameters": { - "type": "object", - "additionalProperties": False, - "required": ["ticket_id", "customer_context"], - "properties": { - "ticket_id": { - "type": "string", - "description": "Unique identifier for the support ticket", - }, - "customer_context": { - "type": ["object", "null"], - "description": "Context about the paid customer, if available", - "additionalProperties": False, - "required": ["user_id", "plan"], - "properties": { - "user_id": { - "type": "string", - "description": "Internal user identifier", - }, - "plan": { - "type": "string", - "description": "Subscription plan name (e.g. pro, enterprise)", - }, - }, - }, - }, - }, - }, - } - ] - - response = litellm.completion( - model="vertex_ai/gemini-3.5-flash", - messages=[{"role": "user", "content": "call the tool"}], - tools=tools, - tool_choice="required", - vertex_location="global", - ) - - print(response) - - -def test_vertex_ai_response_id(): - """Test that litellm preserves the response ID from Vertex AI's API for non-streaming responses""" - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - load_vertex_ai_credentials() - - client = HTTPHandler() - - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - mock_response.json.return_value = { - "responseId": "vertex_ai_response_123", - "candidates": [ - { - "content": { - "role": "model", - "parts": [{"text": "Hello! How can I help you today?"}], - }, - "finishReason": "STOP", - "safetyRatings": [ - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "probability": "NEGLIGIBLE", - } - ], - } - ], - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 8, - "totalTokenCount": 18, - }, - } - - with patch.object(client, "post", return_value=mock_response) as mock_post: - response = completion( - model="vertex_ai/gemini-1.5-pro", - messages=[{"role": "user", "content": "Hi!"}], - client=client, - ) - - # Verify the response ID is preserved - assert response.id == "vertex_ai_response_123" - assert response.choices[0].message.content == "Hello! How can I help you today?" - - - - -def test_vertex_ai_gemini_2_5_pro_streaming(): - try: - load_vertex_ai_credentials() - # litellm.turn_on_debug() - response = completion( - model="vertex_ai/gemini-2.5-pro", - messages=[{"role": "user", "content": "Hi!"}], - vertex_location="global", - stream=True, - ) - has_real_content = False - for chunk in response: - print(chunk) - if ( - chunk.choices[0].delta.content is not None - and len(chunk.choices[0].delta.content) > 0 - ): - has_real_content = True - assert has_real_content - except litellm.RateLimitError: - pytest.skip("Skipping due to rate limit error") - - - - -@pytest.mark.asyncio -async def test_vertex_ai_deepseek(): - """Test that deepseek models use the correct v1 API endpoint instead of v1beta1.""" - load_vertex_ai_credentials() - litellm.turn_on_debug() - from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler - - client = AsyncHTTPHandler() - - # Create a proper mock response - mock_response = MagicMock() - mock_response.json.return_value = { - "choices": [ - { - "message": { - "role": "assistant", - "content": "Hello! How can I help you today?", - }, - "index": 0, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, - "model": "deepseek-ai/deepseek-r1-0528-maas", - } - mock_response.status_code = 200 - - with patch.object(client, "post", return_value=mock_response) as mock_post: - response = await acompletion( - model="vertex_ai/deepseek-ai/deepseek-r1-0528-maas", - messages=[{"role": "user", "content": "Hi!"}], - client=client, - ) - - mock_post.assert_called_once() - # Access the URL from kwargs since the call is made with keyword arguments - url = mock_post.call_args.kwargs["url"] - print(f"mock_post.call_args.kwargs['url']: {url}") - assert "v1beta1" not in url - assert "v1" in url - - -def test_gemini_grounding_on_streaming(): - from litellm import completion - - load_vertex_ai_credentials() - # litellm.turn_on_debug() - args = { - "model": "vertex_ai/gemini-3-flash-preview", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What is the weather like on San Francisco today ?", - } - ], - } - ], - "vertex_location": "global", - "stream": True, - "tools": [{"googleSearch": {}}], - "fallbacks": [], - } - - result = completion(**args) - vertex_ai_grounding_metadata_shows_up = False - for chunk in result: - if hasattr(chunk, "vertex_ai_grounding_metadata"): - vertex_ai_grounding_metadata_shows_up = True - print(chunk) - assert vertex_ai_grounding_metadata_shows_up - - -def test_gemini_google_maps_tool_simple(): - """ - Test googleMaps tool with just enableWidget parameter. - """ - load_vertex_ai_credentials() - litellm.turn_on_debug() - - tools = [{"googleMaps": {"enableWidget": True}}] - tools_with_location = [ - { - "googleMaps": { - "enableWidget": True, - "latitude": 37.7749, - "longitude": -122.4194, - "languageCode": "en_US", - } - } - ] - try: - for tools in [tools, tools_with_location]: - response = completion( - model="vertex_ai/gemini-3-flash-preview", - messages=[ - { - "role": "user", - "content": "What restaurants are nearby?", - } - ], - tools=tools, - vertex_location="global", - ) - print(f"Response: {response.model_dump_json(indent=4)}") - assert response.choices[0].message.content is not None - except (litellm.RateLimitError, litellm.InternalServerError) as e: - pytest.skip(f"Transient Vertex-side failure, not a LiteLLM bug: {e}") - except Exception as e: - pytest.fail(f"Error occurred: {e}") diff --git a/tests/local_testing/test_anthropic_prompt_caching.py b/tests/local_testing/test_anthropic_prompt_caching.py deleted file mode 100644 index 169e6762ad0..00000000000 --- a/tests/local_testing/test_anthropic_prompt_caching.py +++ /dev/null @@ -1,379 +0,0 @@ -import json -import os -import traceback - -from dotenv import load_dotenv - -load_dotenv() -import io -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest -from test_amazing_vertex_completion import load_vertex_ai_credentials -from test_streaming import streaming_format_tests - -import litellm -from litellm import RateLimitError, Timeout, completion, completion_cost, embedding -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler - -# litellm.num_retries =3 -litellm.cache = None -litellm.success_callback = [] -user_message = "Write a short poem about the sky" -messages = [{"content": user_message, "role": "user"}] - - -def logger_fn(user_model_dict): - print(f"user_model_dict: {user_model_dict}") - - -@pytest.fixture(autouse=True) -def reset_callbacks(): - print("\npytest fixture - resetting callbacks") - litellm.success_callback = [] - litellm._async_success_callback = [] - litellm.failure_callback = [] - litellm.callbacks = [] - - -@pytest.fixture -def anthropic_messages(): - return [ - # System Message - { - "role": "system", - "content": [ - { - "type": "text", - "text": "Here is the full text of a complex legal agreement" * 500, - "cache_control": {"type": "ephemeral"}, - } - ], - }, - # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache. - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What are the key terms and conditions in this agreement?", - "cache_control": {"type": "ephemeral"}, - } - ], - }, - { - "role": "assistant", - "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo", - }, - # The final turn is marked with cache-control, for continuing in followups. - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What are the key terms and conditions in this agreement?", - "cache_control": {"type": "ephemeral"}, - } - ], - }, - ] - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.asyncio -async def test_anthropic_vertex_ai_prompt_caching(anthropic_messages, sync_mode): - litellm.turn_on_debug() - - load_vertex_ai_credentials() - - client = HTTPHandler() if sync_mode else AsyncHTTPHandler() - with patch.object(client, "post", return_value=MagicMock()) as mock_post: - try: - if sync_mode: - response = completion( - model="vertex_ai/claude-3-5-sonnet-v2@20241022 ", - messages=anthropic_messages, - client=client, - ) - else: - response = await litellm.acompletion( - model="vertex_ai/claude-3-5-sonnet-v2@20241022 ", - messages=anthropic_messages, - client=client, - ) - except Exception as e: - print(f"Error: {e}") - - mock_post.assert_called_once() - print(mock_post.call_args.kwargs["headers"]) - assert "anthropic-beta" not in mock_post.call_args.kwargs["headers"] - - -@pytest.mark.flaky(retries=3, delay=2) -@pytest.mark.asyncio() -async def test_anthropic_api_prompt_caching_basic(): - litellm.set_verbose = True - response = await litellm.acompletion( - model="anthropic/claude-sonnet-4-5-20250929", - messages=[ - # System Message - { - "role": "system", - "content": [ - { - "type": "text", - "text": "Here is the full text of a complex legal agreement" - * 100, - "cache_control": {"type": "ephemeral"}, - } - ], - }, - # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache. - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What are the key terms and conditions in this agreement?", - "cache_control": {"type": "ephemeral"}, - } - ], - }, - { - "role": "assistant", - "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo", - }, - # The final turn is marked with cache-control, for continuing in followups. - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What are the key terms and conditions in this agreement?", - "cache_control": {"type": "ephemeral"}, - } - ], - }, - ], - temperature=0.2, - max_tokens=10, - extra_headers={ - "anthropic-version": "2023-06-01", - }, - ) - - print("response=", response) - - assert "cache_read_input_tokens" in response.usage - assert "cache_creation_input_tokens" in response.usage - - # Assert either a cache entry was created or cache was read - changes depending on the anthropic api ttl - assert (response.usage.cache_read_input_tokens > 0) or ( - response.usage.cache_creation_input_tokens > 0 - ) - - -@pytest.mark.flaky(retries=3, delay=2) -@pytest.mark.asyncio() -async def test_anthropic_api_prompt_caching_basic_with_cache_creation(): - from uuid import uuid4 - - random_id = uuid4() - - litellm.set_verbose = True - response = await litellm.acompletion( - model="anthropic/claude-sonnet-4-5-20250929", - messages=[ - # System Message - { - "role": "system", - "content": [ - { - "type": "text", - "text": "Here is the full text of a complex legal agreement {}".format( - random_id - ) - * 100, - "cache_control": {"type": "ephemeral", "ttl": "1h"}, - } - ], - }, - # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache. - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What are the key terms and conditions in this agreement?", - "cache_control": {"type": "ephemeral", "ttl": "5m"}, - } - ], - }, - { - "role": "assistant", - "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo", - }, - # The final turn is marked with cache-control, for continuing in followups. - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What are the key terms and conditions in this agreement?", - "cache_control": {"type": "ephemeral"}, - } - ], - }, - ], - temperature=0.2, - max_tokens=10, - extra_headers={ - "anthropic-version": "2023-06-01", - }, - ) - - print("response=", response) - - assert "cache_read_input_tokens" in response.usage - assert "cache_creation_input_tokens" in response.usage - - # Assert either a cache entry was created or cache was read - changes depending on the anthropic api ttl - assert (response.usage.cache_read_input_tokens > 0) or ( - response.usage.cache_creation_input_tokens > 0 - ) - - -@pytest.mark.flaky(retries=3, delay=2) -@pytest.mark.asyncio() -async def test_anthropic_api_prompt_caching_no_headers(): - litellm.set_verbose = True - response = await litellm.acompletion( - model="anthropic/claude-sonnet-4-5-20250929", - messages=[ - # System Message - { - "role": "system", - "content": [ - { - "type": "text", - "text": "Here is the full text of a complex legal agreement" - * 100, - "cache_control": {"type": "ephemeral"}, - } - ], - }, - # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache. - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What are the key terms and conditions in this agreement?", - "cache_control": {"type": "ephemeral"}, - } - ], - }, - { - "role": "assistant", - "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo", - }, - # The final turn is marked with cache-control, for continuing in followups. - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What are the key terms and conditions in this agreement?", - "cache_control": {"type": "ephemeral"}, - } - ], - }, - ], - temperature=0.2, - max_tokens=10, - ) - - print("response=", response) - - assert "cache_read_input_tokens" in response.usage - assert "cache_creation_input_tokens" in response.usage - - # Assert either a cache entry was created or cache was read - changes depending on the anthropic api ttl - assert (response.usage.cache_read_input_tokens > 0) or ( - response.usage.cache_creation_input_tokens > 0 - ) - - -@pytest.mark.flaky(retries=3, delay=2) -@pytest.mark.asyncio() -async def test_anthropic_api_prompt_caching_streaming(): - response = await litellm.acompletion( - model="anthropic/claude-sonnet-4-5-20250929", - messages=[ - # System Message - { - "role": "system", - "content": [ - { - "type": "text", - "text": "Here is the full text of a complex legal agreement" - * 100, - "cache_control": {"type": "ephemeral"}, - } - ], - }, - # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache. - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What are the key terms and conditions in this agreement?", - "cache_control": {"type": "ephemeral"}, - } - ], - }, - { - "role": "assistant", - "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo", - }, - # The final turn is marked with cache-control, for continuing in followups. - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What are the key terms and conditions in this agreement?", - "cache_control": {"type": "ephemeral"}, - } - ], - }, - ], - temperature=0.2, - max_tokens=10, - stream=True, - stream_options={"include_usage": True}, - ) - - idx = 0 - is_cache_read_input_tokens_in_usage = False - is_cache_creation_input_tokens_in_usage = False - async for chunk in response: - streaming_format_tests(idx=idx, chunk=chunk) - # Assert either a cache entry was created or cache was read - changes depending on the anthropic api ttl - if hasattr(chunk, "usage"): - print("Received final usage - {}".format(chunk.usage)) - if hasattr(chunk, "usage") and hasattr(chunk.usage, "cache_read_input_tokens"): - is_cache_read_input_tokens_in_usage = True - if hasattr(chunk, "usage") and hasattr( - chunk.usage, "cache_creation_input_tokens" - ): - is_cache_creation_input_tokens_in_usage = True - - idx += 1 - - print("response=", response) - - assert ( - is_cache_read_input_tokens_in_usage and is_cache_creation_input_tokens_in_usage - ) diff --git a/tests/local_testing/test_async_fn.py b/tests/local_testing/test_async_fn.py deleted file mode 100644 index 8cd03fa258f..00000000000 --- a/tests/local_testing/test_async_fn.py +++ /dev/null @@ -1,93 +0,0 @@ -#### What this tests #### -# This tests the the acompletion function # - -import asyncio - -import pytest - -import litellm -from litellm import acompletion - -litellm.num_retries = 3 - - - - -# test_sync_response_anyscale() - - -def test_async_response_openai(): - - litellm.set_verbose = True - - async def test_get_response(): - user_message = "Hello, how are you?" - messages = [{"content": user_message, "role": "user"}] - tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - }, - }, - "required": ["location"], - }, - }, - } - ] - try: - response = await acompletion( - model="gpt-3.5-turbo", - messages=messages, - tools=tools, - parallel_tool_calls=True, - timeout=5, - ) - print(f"response: {response}") - print(f"response ms: {response._response_ms}") - except litellm.Timeout as e: - pass - except Exception as e: - pytest.fail(f"An exception occurred: {e}") - print(e) - - asyncio.run(test_get_response()) - - -# test_async_response_openai() - - - - -# test_async_anyscale_response() - - - - -# test_async_completion_cloudflare() - - - - - - -# test_get_cloudflare_response_streaming() - - -# test_get_response_streaming() - - - - -# test_get_response_non_openai_streaming() diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py deleted file mode 100644 index b5703224eb1..00000000000 --- a/tests/local_testing/test_completion.py +++ /dev/null @@ -1,1985 +0,0 @@ -import json -import os - -from dotenv import load_dotenv - -load_dotenv() -from unittest.mock import MagicMock, patch - -import httpx -import pytest -from openai import OpenAI - -import litellm -from litellm import Timeout, completion, completion_cost -from litellm.llms.custom_httpx.http_handler import HTTPHandler -from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE - -# litellm.num_retries=3 - -litellm.cache = None -litellm.success_callback = [] -user_message = "Write a short poem about the sky" -messages = [{"content": user_message, "role": "user"}] - - - - -@pytest.fixture(autouse=True) -def reset_callbacks(): - print("\npytest fixture - resetting callbacks") - litellm.success_callback = [] - litellm._async_success_callback = [] - litellm.failure_callback = [] - litellm.callbacks = [] - - - - - - - - -def predibase_mock_post(url, data=None, json=None, headers=None, timeout=None): - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - mock_response.json.return_value = { - "generated_text": " Is it to find happiness, to achieve success,", - "details": { - "finish_reason": "length", - "prompt_tokens": 8, - "generated_tokens": 10, - "seed": None, - "prefill": [], - "tokens": [ - {"id": 2209, "text": " Is", "logprob": -1.7568359, "special": False}, - {"id": 433, "text": " it", "logprob": -0.2220459, "special": False}, - {"id": 311, "text": " to", "logprob": -0.6928711, "special": False}, - {"id": 1505, "text": " find", "logprob": -0.6425781, "special": False}, - { - "id": 23871, - "text": " happiness", - "logprob": -0.07519531, - "special": False, - }, - {"id": 11, "text": ",", "logprob": -0.07110596, "special": False}, - {"id": 311, "text": " to", "logprob": -0.79296875, "special": False}, - { - "id": 11322, - "text": " achieve", - "logprob": -0.7602539, - "special": False, - }, - { - "id": 2450, - "text": " success", - "logprob": -0.03656006, - "special": False, - }, - {"id": 11, "text": ",", "logprob": -0.0011510849, "special": False}, - ], - }, - } - return mock_response - - -# test_completion_predibase() - - -# test_completion_claude() - - - - -@pytest.mark.asyncio -async def test_anthropic_no_content_error(): - """ - https://github.com/BerriAI/litellm/discussions/3440#discussioncomment-9323402 - """ - try: - litellm.drop_params = True - response = await litellm.acompletion( - model="anthropic/claude-sonnet-4-5-20250929", - api_key=os.getenv("ANTHROPIC_API_KEY"), - messages=[ - { - "role": "system", - "content": "You will be given a list of fruits. Use the submitFruit function to submit a fruit. Don't say anything after.", - }, - {"role": "user", "content": "I like apples"}, - { - "content": "The most relevant tool for this request is the submitFruit function.", - "role": "assistant", - "tool_calls": [ - { - "function": { - "arguments": '{"name": "Apple"}', - "name": "submitFruit", - }, - "id": "toolu_012ZTYKWD4VqrXGXyE7kEnAK", - "type": "function", - } - ], - }, - { - "role": "tool", - "content": '{"success":true}', - "tool_call_id": "toolu_012ZTYKWD4VqrXGXyE7kEnAK", - }, - ], - max_tokens=2000, - temperature=1, - tools=[ - { - "type": "function", - "function": { - "name": "submitFruit", - "description": "Submits a fruit", - "parameters": { - "type": "object", - "properties": { - "name": { - "type": "string", - "description": "The name of the fruit", - } - }, - "required": ["name"], - }, - }, - } - ], - frequency_penalty=0.8, - ) - - pass - except litellm.InternalServerError: - pass - except litellm.APIError as e: - if e.status_code != 500: - raise - except Exception as e: - pytest.fail(f"An unexpected error occurred - {str(e)}") - - - - -def encode_image(image_path): - import base64 - - with open(image_path, "rb") as image_file: - return base64.b64encode(image_file.read()).decode("utf-8") - - -@pytest.mark.parametrize( - "model", - [ - "gpt-4o", - "azure/gpt-4.1-mini", - "anthropic/claude-sonnet-4-5-20250929", - ], -) # -def test_completion_base64(model): - try: - import base64 - - import requests - - litellm.set_verbose = True - url = "https://dummyimage.com/100/100/fff&text=Test+image" - response = requests.get(url) - file_data = response.content - - encoded_file = base64.b64encode(file_data).decode("utf-8") - base64_image = f"data:image/png;base64,{encoded_file}" - resp = litellm.completion( - model=model, - messages=[ - { - "role": "user", - "content": [ - {"type": "text", "text": "Whats in this image?"}, - { - "type": "image_url", - "image_url": {"url": base64_image}, - }, - ], - } - ], - ) - print(f"\nResponse: {resp}") - - prompt_tokens = resp.usage.prompt_tokens - except litellm.ServiceUnavailableError as e: - print("got service unavailable error: ", e) - pass - except litellm.InternalServerError as e: - print("got internal server error: ", e) - pass - except Exception as e: - if "500 Internal error encountered.'" in str(e): - pass - else: - pytest.fail(f"An exception occurred - {str(e)}") - - -def test_completion_mistral_api(): - try: - litellm.set_verbose = True - response = completion( - model="mistral/mistral-tiny", - max_tokens=5, - messages=[ - { - "role": "user", - "content": "Hey, how's it going?", - } - ], - seed=10, - ) - # Add any assertions here to check the response - print(response) - - cost = litellm.completion_cost(completion_response=response) - print("cost to make mistral completion=", cost) - assert cost > 0.0 - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - - - -def test_completion_mistral_api_mistral_large_function_call(): - litellm.set_verbose = True - tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - }, - } - ] - messages = [ - { - "role": "user", - "content": "What's the weather like in Boston today in Fahrenheit?", - } - ] - try: - # test without max tokens - response = completion( - model="mistral/mistral-medium-latest", - messages=messages, - tools=tools, - tool_choice="auto", - ) - # Add any assertions, here to check response args - print(response) - assert isinstance(response.choices[0].message.tool_calls[0].function.name, str) - assert isinstance( - response.choices[0].message.tool_calls[0].function.arguments, str - ) - - messages.append( - response.choices[0].message.model_dump() - ) # Add assistant tool invokes - tool_result = ( - '{"location": "Boston", "temperature": "72", "unit": "fahrenheit"}' - ) - # Add user submitted tool results in the OpenAI format - messages.append( - { - "tool_call_id": response.choices[0].message.tool_calls[0].id, - "role": "tool", - "name": response.choices[0].message.tool_calls[0].function.name, - "content": tool_result, - } - ) - # In the second response, Mistral should deduce answer from tool results - second_response = completion( - model="mistral/mistral-large-latest", - messages=messages, - tools=tools, - tool_choice="auto", - ) - print(second_response) - except litellm.RateLimitError: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - - - -# test_completion_mistral_api() - - -def test_completion_mistral_api_modified_input(): - try: - litellm.set_verbose = True - response = completion( - model="mistral/mistral-tiny", - max_tokens=5, - messages=[ - { - "role": "user", - "content": [{"type": "text", "text": "Hey, how's it going?"}], - } - ], - ) - # Add any assertions here to check the response - print(response) - - cost = litellm.completion_cost(completion_response=response) - print("cost to make mistral completion=", cost) - assert cost > 0.0 - except Exception as e: - if "500" in str(e): - pass - else: - pytest.fail(f"Error occurred: {e}") - - - - -# test_completion_azure_gpt4_vision() - - -def test_completion_openai_response_headers(): - """ - Tests if LiteLLM reurns response hea - """ - litellm.return_response_headers = True - - # /chat/completion - messages = [ - { - "role": "user", - "content": "hi", - } - ] - - response = completion( - model="gpt-4o-mini", - messages=messages, - ) - - print(f"response: {response}") - - print("response_headers=", response._response_headers) - assert response._response_headers is not None - assert "x-ratelimit-remaining-tokens" in response._response_headers - assert isinstance( - response._hidden_params["additional_headers"][ - "llm_provider-x-ratelimit-remaining-requests" - ], - str, - ) - - # /chat/completion - with streaming - - streaming_response = litellm.completion( - model="gpt-4o-mini", - messages=messages, - stream=True, - ) - response_headers = streaming_response._response_headers - print("streaming response_headers=", response_headers) - assert response_headers is not None - assert "x-ratelimit-remaining-tokens" in response_headers - assert isinstance( - response._hidden_params["additional_headers"][ - "llm_provider-x-ratelimit-remaining-requests" - ], - str, - ) - - for chunk in streaming_response: - print("chunk=", chunk) - - # embedding - embedding_response = litellm.embedding( - model="text-embedding-ada-002", - input="hello", - ) - - embedding_response_headers = embedding_response._response_headers - print("embedding_response_headers=", embedding_response_headers) - assert embedding_response_headers is not None - assert "x-ratelimit-remaining-tokens" in embedding_response_headers - assert isinstance( - response._hidden_params["additional_headers"][ - "llm_provider-x-ratelimit-remaining-requests" - ], - str, - ) - - litellm.return_response_headers = False - - -@pytest.mark.asyncio() -async def test_async_completion_openai_response_headers(): - """ - Tests if LiteLLM reurns response hea - """ - litellm.return_response_headers = True - - # /chat/completion - messages = [ - { - "role": "user", - "content": "hi", - } - ] - - response = await litellm.acompletion( - model="gpt-4o-mini", - messages=messages, - ) - - print(f"response: {response}") - - print("response_headers=", response._response_headers) - assert response._response_headers is not None - assert "x-ratelimit-remaining-tokens" in response._response_headers - - # /chat/completion with streaming - - streaming_response = await litellm.acompletion( - model="gpt-4o-mini", - messages=messages, - stream=True, - ) - response_headers = streaming_response._response_headers - print("streaming response_headers=", response_headers) - assert response_headers is not None - assert "x-ratelimit-remaining-tokens" in response_headers - - async for chunk in streaming_response: - print("chunk=", chunk) - - # embedding - embedding_response = await litellm.aembedding( - model="text-embedding-ada-002", - input="hello", - ) - - embedding_response_headers = embedding_response._response_headers - print("embedding_response_headers=", embedding_response_headers) - assert embedding_response_headers is not None - assert "x-ratelimit-remaining-tokens" in embedding_response_headers - - litellm.return_response_headers = False - - -@pytest.mark.parametrize("model", ["gpt-3.5-turbo", "gpt-4", "gpt-4o"]) -def test_completion_openai_params(model): - litellm.drop_params = True - messages = [ - { - "role": "user", - "content": """Generate JSON about Bill Gates: { "full_name": "", "title": "" }""", - } - ] - - response = completion( - model=model, - messages=messages, - response_format={"type": "json_object"}, - ) - - print(f"response: {response}") - - -def test_completion_fireworks_ai(): - """ - Mocked so it does not depend on Fireworks' rotating serverless catalog - (no externally-verifiable model list exists). Asserts the request is - built correctly and the OpenAI-compatible response is parsed back. - """ - litellm.set_verbose = True - messages = [ - {"role": "system", "content": "You're a good bot"}, - {"role": "user", "content": "Hey"}, - ] - - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"content-type": "application/json"} - mock_response.json.return_value = { - "id": "chatcmpl-test", - "object": "chat.completion", - "created": 1234567890, - "model": "accounts/fireworks/models/deepseek-v3p1", - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "Hello there!"}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12}, - } - mock_response.text = json.dumps(mock_response.json.return_value) - - client = HTTPHandler() - with patch.object(client, "post", return_value=mock_response) as mock_post: - response = completion( - model="fireworks_ai/accounts/fireworks/models/deepseek-v3p1", - messages=messages, - client=client, - ) - - mock_post.assert_called_once() - request_body = json.loads(mock_post.call_args.kwargs["data"]) - assert "deepseek-v3p1" in request_body["model"] - assert request_body["messages"] == messages - assert response.choices[0].message.content == "Hello there!" - assert response.usage.total_tokens == 12 - - -@pytest.mark.parametrize( - "api_key, api_base", [(None, "my-bad-api-base"), ("my-bad-api-key", None)] -) -def test_completion_fireworks_ai_dynamic_params(api_key, api_base): - try: - litellm.set_verbose = True - messages = [ - {"role": "system", "content": "You're a good bot"}, - { - "role": "user", - "content": "Hey", - }, - ] - response = completion( - model="fireworks_ai/accounts/fireworks/models/mixtral-8x7b-instruct", - messages=messages, - api_base=api_base, - api_key=api_key, - ) - pytest.fail(f"This call should have failed!") - except Exception as e: - pass - - - - -# test_completion_perplexity_api() - - - - -# test_completion_perplexity_api_2() - -# commenting out as this is a flaky test on circle-ci -# def test_completion_nlp_cloud(): -# try: -# messages = [ -# {"role": "system", "content": "You are a helpful assistant."}, -# { -# "role": "user", -# "content": "how does a court case get to the Supreme Court?", -# }, -# ] -# response = completion(model="dolphin", messages=messages, logger_fn=logger_fn) -# print(response) -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") - -# test_completion_nlp_cloud() - -######### HUGGING FACE TESTS ######################## -##################################################### -""" -HF Tests we should pass -- TGI: - - Pro Inference API - - Deployed Endpoint -- Coversational - - Free Inference API - - Deployed Endpoint -- Neither TGI or Coversational - - Free Inference API - - Deployed Endpoint -""" - - - - - - -def test_lm_studio_completion(monkeypatch): - monkeypatch.delenv("LM_STUDIO_API_KEY", raising=False) - monkeypatch.delenv("OPENAI_API_KEY", raising=False) - litellm.turn_on_debug() - try: - completion( - api_key="fake-key", - model="lm_studio/typhoon2-quen2.5-7b-instruct", - messages=[ - {"role": "user", "content": "What's the weather like in San Francisco?"} - ], - api_base=FAKE_OPENAI_API_BASE, - ) - except litellm.AuthenticationError as e: - pytest.fail(f"Error occurred: {e}") - except litellm.APIError as e: - print(e) - - -# ################### Hugging Face Conversational models ######################## -# def hf_test_completion_conv(): -# try: -# response = litellm.completion( -# model="huggingface/facebook/blenderbot-3B", -# messages=[{ "content": "Hello, how are you?","role": "user"}], -# ) -# # Add any assertions here to check the response -# print(response) -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") -# hf_test_completion_conv() - -# ################### Hugging Face Neither TGI or Conversational models ######################## -# # Neither TGI or Conversational task -# def hf_test_completion_none_task(): -# try: -# user_message = "My name is Merve and my favorite" -# messages = [{ "content": user_message,"role": "user"}] -# response = completion( -# model="huggingface/roneneldan/TinyStories-3M", -# messages=messages, -# api_base="https://p69xlsj6rpno5drq.us-east-1.aws.endpoints.huggingface.cloud", -# ) -# # Add any assertions here to check the response -# print(response) -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") -# hf_test_completion_none_task() - - -def mock_post(url, **kwargs): - print(f"url={url}") - if "text-classification" in url: - raise Exception("Model not found") - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - mock_response.json.return_value = [ - [ - {"label": "LABEL_0", "score": 0.9990691542625427}, - {"label": "LABEL_1", "score": 0.0009308889275416732}, - ] - ] - return mock_response - - - - -########################### End of Hugging Face Tests ############################################## -# def test_completion_hf_api(): -# # failing on circle-ci commenting out -# try: -# user_message = "write some code to find the sum of two numbers" -# messages = [{ "content": user_message,"role": "user"}] -# api_base = "https://a8l9e3ucxinyl3oj.us-east-1.aws.endpoints.huggingface.cloud" -# response = completion(model="huggingface/meta-llama/Llama-2-7b-chat-hf", messages=messages, api_base=api_base) -# # Add any assertions here to check the response -# print(response) -# except Exception as e: -# if "loading" in str(e): -# pass -# pytest.fail(f"Error occurred: {e}") - -# test_completion_hf_api() - -# def test_completion_hf_api_best_of(): -# # failing on circle ci commenting out -# try: -# user_message = "write some code to find the sum of two numbers" -# messages = [{ "content": user_message,"role": "user"}] -# api_base = "https://a8l9e3ucxinyl3oj.us-east-1.aws.endpoints.huggingface.cloud" -# response = completion(model="huggingface/meta-llama/Llama-2-7b-chat-hf", messages=messages, api_base=api_base, n=2) -# # Add any assertions here to check the response -# print(response) -# except Exception as e: -# if "loading" in str(e): -# pass -# pytest.fail(f"Error occurred: {e}") - -# test_completion_hf_api_best_of() - -# def test_completion_hf_deployed_api(): -# try: -# user_message = "There's a llama in my garden 😱 What should I do?" -# messages = [{ "content": user_message,"role": "user"}] -# response = completion(model="huggingface/https://ji16r2iys9a8rjk2.us-east-1.aws.endpoints.huggingface.cloud", messages=messages, logger_fn=logger_fn) -# # Add any assertions here to check the response -# print(response) -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") - - -# this should throw an exception, to trigger https://logs.litellm.ai/ -# def hf_test_error_logs(): -# try: -# litellm.set_verbose=True -# user_message = "My name is Merve and my favorite" -# messages = [{ "content": user_message,"role": "user"}] -# response = completion( -# model="huggingface/roneneldan/TinyStories-3M", -# messages=messages, -# api_base="https://p69xlsj6rpno5drq.us-east-1.aws.endpoints.huggingface.cloud", - -# ) -# # Add any assertions here to check the response -# print(response) - -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") - -# hf_test_error_logs() - - -def test_completion_openai(): - try: - litellm.set_verbose = True - litellm.drop_params = True - print(f"api key: {os.environ['OPENAI_API_KEY']}") - litellm.api_key = os.environ["OPENAI_API_KEY"] - response = completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hey"}], - max_tokens=10, - metadata={"hi": "bye"}, - ) - print("This is the response object\n", response) - - response_str = response["choices"][0]["message"]["content"] - response_str_2 = response.choices[0].message.content - - cost = completion_cost(completion_response=response) - print("Cost for completion call with gpt-3.5-turbo: ", f"${float(cost):.10f}") - assert response_str == response_str_2 - assert type(response_str) == str - assert len(response_str) > 1 - - litellm.api_key = None - except Timeout as e: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -@pytest.mark.parametrize( - "model, api_version", - [ - # ("gpt-4o-2024-08-06", None), - # ("azure/gpt-4.1-mini", None), - ("bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", None), - # ("azure/gpt-4o-new-test", "2024-08-01-preview"), - ], -) -@pytest.mark.flaky(retries=3, delay=1) -def test_completion_openai_pydantic(model, api_version): - try: - litellm.turn_on_debug() - from pydantic import BaseModel - - messages = [ - {"role": "user", "content": "List 5 important events in the XIX century"} - ] - - class CalendarEvent(BaseModel): - name: str - date: str - participants: list[str] - - class EventsList(BaseModel): - events: list[CalendarEvent] - - litellm.enable_json_schema_validation = True - for _ in range(3): - try: - response = completion( - model=model, - messages=messages, - metadata={"hi": "bye"}, - response_format=EventsList, - api_version=api_version, - ) - break - except litellm.JSONSchemaValidationError: - pytest.fail("ERROR OCCURRED! INVALID JSON") - - print("This is the response object\n", response) - - response_str = response["choices"][0]["message"]["content"] - - print(f"response_str: {response_str}") - json.loads(response_str) # check valid json is returned - - except Timeout: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -def test_completion_text_openai(): - try: - # litellm.set_verbose =True - response = completion(model="text-completion-openai/gpt-5.4-nano", messages=messages) - print(response["choices"][0]["message"]["content"]) - except Exception as e: - print(e) - pytest.fail(f"Error occurred: {e}") - - -@pytest.mark.asyncio -async def test_completion_text_openai_async(): - try: - # litellm.set_verbose =True - response = await litellm.acompletion( - model="text-completion-openai/gpt-5.4-nano", messages=messages - ) - print(response["choices"][0]["message"]["content"]) - except Exception as e: - print(e) - pytest.fail(f"Error occurred: {e}") - - -def test_completion_openai_with_optional_params(): - # [Proxy PROD TEST] WARNING: DO NOT DELETE THIS TEST - on_request = MagicMock() - client = OpenAI(http_client=httpx.Client(event_hooks={"request": [on_request]})) - response = completion( - model="gpt-6-luna", - reasoning_effort="none", - messages=[{"role": "user", "content": "respond in valid, json - what is the day"}], - temperature=0.5, - top_p=0.1, - seed=12, - response_format={"type": "json_object"}, - logit_bias=None, - user="ishaans app", - client=client, - ) - - assert response.choices[0].message.content - on_request.assert_called_once() - sent = json.loads(on_request.call_args.args[0].content) - assert sent["model"] == "gpt-6-luna" - assert sent["user"] == "ishaans app" - assert sent["seed"] == 12 - assert sent["temperature"] == 0.5 - assert sent["top_p"] == 0.1 - assert sent["response_format"] == {"type": "json_object"} - assert "logit_bias" not in sent - - -# test_completion_openai_with_optional_params() - - -def test_completion_logprobs(): - """ - This function is used to test the litellm.completion logprobs functionality. - - Parameters: - None - - Returns: - None - """ - try: - litellm.set_verbose = True - response = completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "what is the time"}], - temperature=0.5, - top_p=0.1, - seed=12, - logit_bias=None, - user="ishaans app", - logprobs=True, - top_logprobs=3, - ) - # Add any assertions here to check the response - - print(response) - print(len(response.choices[0].logprobs["content"][0]["top_logprobs"])) - assert "logprobs" in response.choices[0] - assert "content" in response.choices[0]["logprobs"] - assert len(response.choices[0].logprobs["content"][0]["top_logprobs"]) == 3 - - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_completion_logprobs() - - -def test_completion_logprobs_stream(): - """ - This function is used to test the litellm.completion logprobs functionality. - - Parameters: - None - - Returns: - None - """ - try: - litellm.set_verbose = False - response = completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "what is the time"}], - temperature=0.5, - top_p=0.1, - seed=12, - max_tokens=5, - logit_bias=None, - user="ishaans app", - logprobs=True, - top_logprobs=3, - stream=True, - ) - # Add any assertions here to check the response - - print(response) - - found_logprob = False - for chunk in response: - # check if atleast one chunk has log probs - print(chunk) - if len(chunk.choices) == 0: - continue - print(f"chunk.choices[0]: {chunk.choices[0]}") - if ( - "logprobs" in chunk.choices[0] - and chunk.choices[0].logprobs is not None - and len(chunk.choices[0].logprobs.content) > 0 - ): - # assert we got a valid logprob in the choices - assert len(chunk.choices[0].logprobs.content[0].top_logprobs) == 3 - found_logprob = True - break - print(chunk) - assert found_logprob == True - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_completion_logprobs_stream() - - -def test_completion_openai_litellm_key(): - try: - litellm.set_verbose = True - litellm.num_retries = 0 - litellm.api_key = os.environ["OPENAI_API_KEY"] - - # ensure key is set to None in .env and in openai.api_key - os.environ["OPENAI_API_KEY"] = "" - import openai - - openai.api_key = "" - ########################################################## - - response = completion( - model="gpt-3.5-turbo", - messages=messages, - temperature=0.5, - top_p=0.1, - max_tokens=10, - user="ishaan_dev@berri.ai", - ) - # Add any assertions here to check the response - print(response) - - ###### reset environ key - os.environ["OPENAI_API_KEY"] = litellm.api_key - - ##### unset litellm var - litellm.api_key = None - except Timeout as e: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_ completion_openai_litellm_key() - - - - -# test_completion_ollama_hosted() - - - - - - - - - - -def test_completion_openrouter_reasoning_effort(): - try: - litellm.set_verbose = True - response = completion( - model="openrouter/deepseek/deepseek-r1", - messages=messages, - include_reasoning=True, - max_tokens=5, - ) - # Add any assertions here to check the response - print(response) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_completion_openrouter1() - - - - -# test_completion_hf_model_no_provider() - - - - - - -# test_completion_anyscale_with_functions() - - -def test_completion_azure_extra_headers(): - # this tests if we can pass api_key to completion, when it's not in the env. - # DO NOT REMOVE THIS TEST. No MATTER WHAT Happens! - # If you want to remove it, speak to Ishaan! - # Ishaan will be very disappointed if this test is removed -> this is a standard way to pass api_key + the router + proxy use this - from httpx import Client - - - http_client = Client() - - with patch.object(http_client, "send", new=MagicMock()) as mock_client: - litellm.client_session = http_client - try: - response = completion( - model="azure/gpt-4.1-mini", - messages=messages, - api_base=os.getenv("AZURE_AI_API_BASE"), - api_version="2023-07-01-preview", - api_key=os.getenv("AZURE_AI_API_KEY"), - extra_headers={ - "Authorization": "my-bad-key", - "Ocp-Apim-Subscription-Key": "hello-world-testing", - }, - ) - print(response) - pytest.fail("Expected this to fail") - except Exception as e: - pass - - mock_client.assert_called() - - print(f"mock_client.call_args: {mock_client.call_args}") - request = mock_client.call_args[0][0] - print(request.method) # This will print 'POST' - print(request.url) # This will print the full URL - print(request.headers) # This will print the full URL - auth_header = request.headers.get("Authorization") - apim_key = request.headers.get("Ocp-Apim-Subscription-Key") - print(auth_header) - assert auth_header == "my-bad-key" - assert apim_key == "hello-world-testing" - - -def test_completion_azure_ad_token(): - # this tests if we can pass api_key to completion, when it's not in the env. - # DO NOT REMOVE THIS TEST. No MATTER WHAT Happens! - # If you want to remove it, speak to Ishaan! - # Ishaan will be very disappointed if this test is removed -> this is a standard way to pass api_key + the router + proxy use this - from httpx import Client - - from litellm import completion - - litellm.set_verbose = True - - old_key = os.environ["AZURE_AI_API_KEY"] - os.environ.pop("AZURE_AI_API_KEY", None) - - http_client = Client() - - with patch.object(http_client, "send", new=MagicMock()) as mock_client: - litellm.client_session = http_client - try: - response = completion( - model="azure/gpt-4.1-mini", - messages=messages, - azure_ad_token="my-special-token", - ) - print(response) - except Exception as e: - pass - finally: - os.environ["AZURE_AI_API_KEY"] = old_key - - mock_client.assert_called_once() - request = mock_client.call_args[0][0] - print(request.method) # This will print 'POST' - print(request.url) # This will print the full URL - print(request.headers) # This will print the full URL - auth_header = request.headers.get("Authorization") - assert auth_header == "Bearer my-special-token" - - -def test_completion_azure_key_completion_arg(): - # this tests if we can pass api_key to completion, when it's not in the env. - # DO NOT REMOVE THIS TEST. No MATTER WHAT Happens! - # If you want to remove it, speak to Ishaan! - # Ishaan will be very disappointed if this test is removed -> this is a standard way to pass api_key + the router + proxy use this - old_key = os.environ["AZURE_AI_API_KEY"] - os.environ.pop("AZURE_AI_API_KEY", None) - try: - print("azure gpt-3.5 test\n\n") - litellm.set_verbose = True - ## Test azure call - response = completion( - model="azure/gpt-4.1-mini", - messages=messages, - api_key=old_key, - logprobs=True, - max_tokens=10, - ) - - print(f"response: {response}") - - print("Hidden Params", response._hidden_params) - assert response._hidden_params["custom_llm_provider"] == "azure" - os.environ["AZURE_AI_API_KEY"] = old_key - except Exception as e: - os.environ["AZURE_AI_API_KEY"] = old_key - pytest.fail(f"Error occurred: {e}") - - -async def test_re_use_azure_async_client(): - try: - print("azure gpt-3.5 ASYNC with clie nttest\n\n") - litellm.set_verbose = True - import openai - - client = openai.AsyncAzureOpenAI( - azure_endpoint=os.environ["AZURE_AI_API_BASE"], - api_key=os.environ["AZURE_AI_API_KEY"], - api_version="2023-07-01-preview", - ) - ## Test azure call - for _ in range(3): - response = await litellm.acompletion( - model="azure/gpt-4.1-mini", messages=messages, client=client - ) - print(f"response: {response}") - except Exception as e: - pytest.fail("got Exception", e) - - - - -# test_azure_openai_ad_token() - - -def test_completion_azure2(): - # test if we can pass api_base, api_version and api_key in compleition() - try: - print("azure gpt-3.5 test\n\n") - litellm.set_verbose = False - api_base = os.environ["AZURE_AI_API_BASE"] - api_key = os.environ["AZURE_AI_API_KEY"] - api_version = os.environ["AZURE_API_VERSION"] - - os.environ["AZURE_AI_API_BASE"] = "" - os.environ["AZURE_API_VERSION"] = "" - os.environ["AZURE_AI_API_KEY"] = "" - - ## Test azure call - response = completion( - model="azure/gpt-4.1-mini", - messages=messages, - api_base=api_base, - api_key=api_key, - api_version=api_version, - max_tokens=10, - ) - - # Add any assertions here to check the response - print(response) - - os.environ["AZURE_AI_API_BASE"] = api_base - os.environ["AZURE_API_VERSION"] = api_version - os.environ["AZURE_AI_API_KEY"] = api_key - - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_completion_azure2() - - -def test_completion_azure3(): - # test if we can pass api_base, api_version and api_key in compleition() - try: - print("azure gpt-3.5 test\n\n") - litellm.set_verbose = True - litellm.api_base = os.environ["AZURE_AI_API_BASE"] - litellm.api_key = os.environ["AZURE_AI_API_KEY"] - litellm.api_version = os.environ["AZURE_API_VERSION"] - - os.environ["AZURE_AI_API_BASE"] = "" - os.environ["AZURE_API_VERSION"] = "" - os.environ["AZURE_AI_API_KEY"] = "" - - ## Test azure call - response = completion( - model="azure/gpt-4.1-mini", - messages=messages, - max_tokens=10, - ) - - # Add any assertions here to check the response - print(response) - - os.environ["AZURE_AI_API_BASE"] = litellm.api_base - os.environ["AZURE_API_VERSION"] = litellm.api_version - os.environ["AZURE_AI_API_KEY"] = litellm.api_key - - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_completion_azure3() - - -# new azure test for using litellm. vars, -# use the following vars in this test and make an azure_api_call -# litellm.api_type = self.azure_api_type -# litellm.api_base = self.AZURE_AI_API_BASE -# litellm.api_version = self.azure_api_version -# litellm.api_key = self.api_key -def test_completion_azure_with_litellm_key(): - try: - print("azure gpt-3.5 test\n\n") - import openai - - #### set litellm vars - litellm.api_type = "azure" - litellm.api_base = os.environ["AZURE_AI_API_BASE"] - litellm.api_version = os.environ["AZURE_API_VERSION"] - litellm.api_key = os.environ["AZURE_AI_API_KEY"] - - ######### UNSET ENV VARs for this ################ - os.environ["AZURE_AI_API_BASE"] = "" - os.environ["AZURE_API_VERSION"] = "" - os.environ["AZURE_AI_API_KEY"] = "" - - ######### UNSET OpenAI vars for this ############## - openai.api_type = "" - openai.api_base = "gm" - openai.api_version = "333" - openai.api_key = "ymca" - - response = completion( - model="azure/gpt-4.1-mini", - messages=messages, - ) - # Add any assertions here to check the response - print(response) - - ######### RESET ENV VARs for this ################ - os.environ["AZURE_AI_API_BASE"] = litellm.api_base - os.environ["AZURE_API_VERSION"] = litellm.api_version - os.environ["AZURE_AI_API_KEY"] = litellm.api_key - - ######### UNSET litellm vars - litellm.api_type = None - litellm.api_base = None - litellm.api_version = None - litellm.api_key = None - - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - - - - - - - -# test_completion_replicate_vicuna() - - -def test_replicate_custom_prompt_dict(): - litellm.set_verbose = True - model_name = "replicate/meta/llama-2-7b" - litellm.register_prompt_template( - model="replicate/meta/llama-2-7b", - initial_prompt_value="You are a good assistant", # [OPTIONAL] - roles={ - "system": { - "pre_message": "[INST] <>\n", # [OPTIONAL] - "post_message": "\n<>\n [/INST]\n", # [OPTIONAL] - }, - "user": { - "pre_message": "[INST] ", # [OPTIONAL] - "post_message": " [/INST]", # [OPTIONAL] - }, - "assistant": { - "pre_message": "\n", # [OPTIONAL] - "post_message": "\n", # [OPTIONAL] - }, - }, - final_prompt_value="Now answer as best you can:", # [OPTIONAL] - ) - try: - response = completion( - model=model_name, - messages=[ - { - "role": "user", - "content": "what is yc write 1 paragraph", - } - ], - mock_response="Hello world", - repetition_penalty=0.1, - num_retries=3, - ) - - except litellm.APIError as e: - pass - except litellm.APIConnectionError as e: - pass - except Exception as e: - pytest.fail(f"An exception occurred - {str(e)}") - print(f"response: {response}") - litellm.custom_prompt_dict = {} # reset - - - - - - -# test_replicate_custom_prompt_dict() - -# commenthing this out since we won't be always testing a custom, replicate deployment -# def test_completion_replicate_deployments(): -# print("TESTING REPLICATE") -# litellm.set_verbose=False -# model_name = "replicate/deployments/ishaan-jaff/ishaan-mistral" -# try: -# response = completion( -# model=model_name, -# messages=messages, -# temperature=0.5, -# seed=-1, -# ) -# print(response) -# # Add any assertions here to check the response -# response_str = response["choices"][0]["message"]["content"] -# print("RESPONSE STRING\n", response_str) -# if type(response_str) != str: -# pytest.fail(f"Error occurred: {e}") -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") -# test_completion_replicate_deployments() - - -######## Test TogetherAI ######## - - -# test_completion_together_ai_mixtral() - - -# test_completion_together_ai_yi_chat() - - -# test_completion_together_ai() - - -def response_format_tests(response: litellm.ModelResponse): - assert isinstance(response.id, str) - assert response.id != "" - - assert isinstance(response.object, str) - assert response.object != "" - - assert isinstance(response.created, int) - - assert isinstance(response.model, str) - assert response.model != "" - - assert isinstance(response.choices, list) - assert len(response.choices) == 1 - choice = response.choices[0] - assert isinstance(choice, litellm.Choices) - assert isinstance(choice.get("index"), int) - - message = choice.get("message") - assert isinstance(message, litellm.Message) - assert isinstance(message.get("role"), str) - assert message.get("role") != "" - assert isinstance(message.get("content"), str) - assert message.get("content") != "" - - assert choice.get("logprobs") is None - assert isinstance(choice.get("finish_reason"), str) - assert choice.get("finish_reason") != "" - - assert isinstance(response.usage, litellm.Usage) # type: ignore - assert isinstance(response.usage.prompt_tokens, int) # type: ignore - assert isinstance(response.usage.completion_tokens, int) # type: ignore - assert isinstance(response.usage.total_tokens, int) # type: ignore - - -@pytest.mark.parametrize( - "model", - [ - "bedrock/mistral.mistral-large-2407-v1:0", - "us.anthropic.claude-sonnet-4-5-20250929-v1:0", - "mistral.mistral-7b-instruct-v0:2", - "meta.llama3-8b-instruct-v1:0", - ], -) -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.asyncio -async def test_completion_bedrock_httpx_models(sync_mode, model): - litellm.set_verbose = True - try: - - if sync_mode: - response = completion( - model=model, - messages=[{"role": "user", "content": "Hey! how's it going?"}], - temperature=0.2, - max_tokens=200, - ) - - assert isinstance(response, litellm.ModelResponse) - - response_format_tests(response=response) - else: - response = await litellm.acompletion( - model=model, - messages=[{"role": "user", "content": "Hey! how's it going?"}], - temperature=0.2, - max_tokens=100, - ) - - assert isinstance(response, litellm.ModelResponse) - - print(f"response: {response}") - response_format_tests(response=response) - - print(f"response: {response}") - except litellm.RateLimitError as e: - print("got rate limit error=", e) - pass - except Exception as e: - pytest.fail(f"An error occurred - {str(e)}") - - -# test_completion_bedrock_titan() - - -# test_completion_bedrock_claude() - - -# def test_completion_bedrock_claude_stream(): -# print("calling claude") -# litellm.set_verbose = False -# try: -# response = completion( -# model="bedrock/anthropic.claude-instant-v1", -# messages=messages, -# stream=True -# ) -# # Add any assertions here to check the response -# print(response) -# for chunk in response: -# print(chunk) -# except RateLimitError: -# pass -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") -# test_completion_bedrock_claude_stream() - - -######## Test VLLM ######## -# def test_completion_vllm(): -# try: -# response = completion( -# model="vllm/facebook/opt-125m", -# messages=messages, -# temperature=0.2, -# max_tokens=80, -# ) -# print(response) -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") - -# test_completion_vllm() - -# def test_completion_hosted_chatCompletion(): -# # this tests calling a server where vllm is hosted -# # this should make an openai.Completion() call to the specified api_base -# # send a request to this proxy server: https://replit.com/@BerriAI/openai-proxy#main.py -# # it checks if model == facebook/opt-125m and returns test passed -# try: -# litellm.set_verbose = True -# response = completion( -# model="facebook/opt-125m", -# messages=messages, -# temperature=0.2, -# max_tokens=80, -# api_base="https://openai-proxy.berriai.repl.co", -# custom_llm_provider="openai" -# ) -# print(response) - -# if response['choices'][0]['message']['content'] != "passed": -# # see https://replit.com/@BerriAI/openai-proxy#main.py -# pytest.fail(f"Error occurred: proxy server did not respond") -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") - -# test_completion_hosted_chatCompletion() - -# def test_completion_custom_api_base(): -# try: -# response = completion( -# model="custom/meta-llama/Llama-2-13b-hf", -# messages=messages, -# temperature=0.2, -# max_tokens=10, -# api_base="https://api.autoai.dev/inference", -# request_timeout=300, -# ) -# # Add any assertions here to check the response -# print("got response\n", response) -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") - -# test_completion_custom_api_base() - - -def test_completion_with_fallbacks(): - print(f"RUNNING TEST COMPLETION WITH FALLBACKS - test_completion_with_fallbacks") - fallbacks = ["gpt-3.5-turbo", "gpt-3.5-turbo", "command-nightly"] - try: - response = completion( - model="bad-model", messages=messages, force_timeout=120, fallbacks=fallbacks - ) - # Add any assertions here to check the response - print(response) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_completion_with_fallbacks() - - -# @pytest.mark.parametrize( -# "function_call", -# [ -# [{"role": "function", "name": "get_capital", "content": "Kokoko"}], -# [ -# {"role": "function", "name": "get_capital", "content": "Kokoko"}, -# {"role": "function", "name": "get_capital", "content": "Kokoko"}, -# ], -# ], -# ) -# @pytest.mark.parametrize( -# "tool_call", -# [ -# [{"role": "tool", "tool_call_id": "1234", "content": "Kokoko"}], -# [ -# {"role": "tool", "tool_call_id": "12344", "content": "Kokoko"}, -# {"role": "tool", "tool_call_id": "1214", "content": "Kokoko"}, -# ], -# ], -# ) - - - - - - -# test_completion_with_fallbacks_multiple_keys() -def test_petals(): - try: - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - client = HTTPHandler() - with patch.object(client, "post") as mock_post: - try: - completion( - model="petals-team/StableBeluga2", - messages=messages, - client=client, - api_base="https://api.petals.dev", - ) - except Exception as e: - print(f"Error occurred: {e}") - mock_post.assert_called_once() - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# def test_baseten(): -# try: - -# response = completion(model="baseten/7qQNLDB", messages=messages, logger_fn=logger_fn) -# # Add any assertions here to check the response -# print(response) -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") - -# test_baseten() -# def test_baseten_falcon_7bcompletion(): -# model_name = "qvv0xeq" -# try: -# response = completion(model=model_name, messages=messages, custom_llm_provider="baseten") -# # Add any assertions here to check the response -# print(response) -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") - -# def test_baseten_falcon_7bcompletion_withbase(): -# model_name = "qvv0xeq" -# litellm.api_base = "https://app.baseten.co" -# try: -# response = completion(model=model_name, messages=messages) -# # Add any assertions here to check the response -# print(response) -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") -# litellm.api_base = None - -# test_baseten_falcon_7bcompletion_withbase() - - -# def test_baseten_wizardLMcompletion_withbase(): -# model_name = "q841o8w" -# litellm.api_base = "https://app.baseten.co" -# try: -# response = completion(model=model_name, messages=messages) -# # Add any assertions here to check the response -# print(response) -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") - -# test_baseten_wizardLMcompletion_withbase() - -# def test_baseten_mosaic_ML_completion_withbase(): -# model_name = "31dxrj3", -# litellm.api_base = "https://app.baseten.co" -# try: -# response = completion(model=model_name, messages=messages) -# # Add any assertions here to check the response -# print(response) -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") - - -# test_completion_ai21() -# test_completion_ai21() -## test deep infra - - -# test_completion_deep_infra() - - - - -# test_completion_deep_infra_mistral() - - - - -# Gemini tests -@pytest.mark.parametrize( - "model", - [ - # "gemini-1.0-pro", - "gemini-2.5-flash-lite", - ], -) -@pytest.mark.flaky(retries=3, delay=1) -def test_completion_gemini(model): - litellm.set_verbose = True - model_name = "gemini/{}".format(model) - messages = [ - {"role": "system", "content": "Be a good bot!"}, - {"role": "user", "content": "Hey, how's it going?"}, - ] - try: - response = completion( - model=model_name, - messages=messages, - safety_settings=[ - { - "category": "HARM_CATEGORY_HARASSMENT", - "threshold": "BLOCK_NONE", - }, - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "threshold": "BLOCK_NONE", - }, - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "threshold": "BLOCK_NONE", - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "threshold": "BLOCK_NONE", - }, - ], - ) - # Add any assertions,here to check the response - print(response) - assert response.choices[0]["index"] == 0 - except litellm.RateLimitError: - pass - except litellm.APIError: - pass - except Exception as e: - if "InternalServerError" in str(e): - pass - else: - pytest.fail(f"Error occurred:{e}") - - -# test_completion_gemini() - - -# Deepseek tests - - - - - - - - - - -# test_completion_palm_stream() - -# test_completion_deep_infra() -# test_completion_ai21() -# test config file with completion # -# def test_completion_openai_config(): -# try: -# litellm.config_path = "../config.json" -# litellm.set_verbose = True -# response = litellm.config_completion(messages=messages) -# # Add any assertions here to check the response -# print(response) -# litellm.config_path = None -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") - - -# def test_maritalk(): -# messages = [{"role": "user", "content": "Hey"}] -# try: -# response = completion("maritalk", messages=messages) -# print(f"response: {response}") -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") -# test_maritalk() - - -def test_moderation(): - response = litellm.moderation(input="i'm ishaan cto of litellm") - print(response) - output = response.results[0] - print(output) - return output - - - - -@pytest.mark.parametrize( - "model", - ["gpt-4o", "azure/gpt-4.1-mini"], -) -@pytest.mark.parametrize( - "stream", - [False, True], -) -@pytest.mark.flaky(retries=3, delay=1) -def test_completion_response_ratelimit_headers(model, stream): - response = completion( - model=model, - messages=[{"role": "user", "content": "Hello world"}], - stream=stream, - ) - hidden_params = response._hidden_params - additional_headers = hidden_params.get("additional_headers", {}) - - print(additional_headers) - for k, v in additional_headers.items(): - assert v != "None" and v is not None - assert "x-ratelimit-remaining-requests" in additional_headers - assert "x-ratelimit-remaining-tokens" in additional_headers - - if model == "azure/gpt-4.1-mini": - # Azure OpenAI header - assert "llm_provider-azureml-model-session" in additional_headers - if model == "claude-3-sonnet-20240229": - # anthropic header - assert "llm_provider-anthropic-ratelimit-requests-reset" in additional_headers - - - - - - - - -def test_langfuse_completion(monkeypatch): - monkeypatch.setenv( - "LANGFUSE_PUBLIC_KEY", "pk-lf-b3db7e8e-c2f6-4fc7-825c-a541a8fbe003" - ) - monkeypatch.setenv( - "LANGFUSE_SECRET_KEY", "sk-lf-b11ef3a8-361c-4445-9652-12318b8596e4" - ) - monkeypatch.setenv("LANGFUSE_HOST", "https://us.cloud.langfuse.com") - litellm.set_verbose = True - resp = litellm.completion( - model="langfuse/gpt-3.5-turbo", - langfuse_public_key=os.getenv("LANGFUSE_PUBLIC_KEY"), - langfuse_secret_key=os.getenv("LANGFUSE_SECRET_KEY"), - langfuse_host="https://us.cloud.langfuse.com", - prompt_id="test-chat-prompt", - prompt_variables={"user_message": "this is used"}, - messages=[{"role": "user", "content": "this is ignored"}], - ) - - -def test_qwen_text_completion(): - # litellm.turn_on_debug() - resp = litellm.completion( - model="text-completion-openai/gpt-5.4-nano", - messages=[{"content": "hello", "role": "user"}], - stream=False, - logprobs=1, - ) - assert resp.choices[0].message.content is not None - assert resp.choices[0].logprobs.token_logprobs[0] is not None - print( - f"resp.choices[0].logprobs.token_logprobs[0]: {resp.choices[0].logprobs.token_logprobs[0]}" - ) - - - - -def test_completion_o3_mini_temperature(): - try: - litellm.set_verbose = True - resp = litellm.completion( - model="o3-mini", - temperature=0.0, - messages=[ - { - "role": "user", - "content": "Hello, world!", - } - ], - drop_params=True, - ) - assert resp.choices[0].message.content is not None - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -def test_completion_gpt_4o_empty_str(): - litellm.turn_on_debug() - from openai import OpenAI - from unittest.mock import MagicMock - - client = OpenAI() - - # Create response object matching OpenAI's format - mock_response_data = { - "id": "chatcmpl-B0W3vmiM78Xkgx7kI7dr7PC949DMS", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "logprobs": None, - "message": { - "content": "", - "refusal": None, - "role": "assistant", - "audio": None, - "function_call": None, - "tool_calls": None, - }, - } - ], - "created": 1739462947, - "model": "gpt-4o-mini-2024-07-18", - "object": "chat.completion", - "service_tier": "default", - "system_fingerprint": "fp_bd83329f63", - "usage": { - "completion_tokens": 1, - "prompt_tokens": 121, - "total_tokens": 122, - "completion_tokens_details": { - "accepted_prediction_tokens": 0, - "audio_tokens": 0, - "reasoning_tokens": 0, - "rejected_prediction_tokens": 0, - }, - "prompt_tokens_details": {"audio_tokens": 0, "cached_tokens": 0}, - }, - } - - # Create a mock response object - mock_raw_response = MagicMock() - mock_raw_response.headers = { - "x-request-id": "123", - "openai-organization": "org-123", - "x-ratelimit-limit-requests": "100", - "x-ratelimit-remaining-requests": "99", - } - mock_raw_response.parse.return_value = mock_response_data - - # Set up the mock completion - mock_completion = MagicMock() - mock_completion.return_value = mock_raw_response - - with patch.object( - client.chat.completions.with_raw_response, "create", mock_completion - ) as mock_create: - resp = litellm.completion( - model="gpt-4o-mini", - messages=[{"role": "user", "content": ""}], - ) - assert resp.choices[0].message.content is not None diff --git a/tests/local_testing/test_completion_cost.py b/tests/local_testing/test_completion_cost.py deleted file mode 100644 index 5cf14cc3241..00000000000 --- a/tests/local_testing/test_completion_cost.py +++ /dev/null @@ -1,466 +0,0 @@ -import asyncio -import json -import os -import time -import traceback -from typing import Optional -from unittest.mock import MagicMock, patch - -import pytest - -import litellm -import litellm.cost_calculator -from litellm import ( - completion_cost, -) -from litellm.litellm_core_utils.litellm_logging import CustomLogger -from litellm.llms.custom_httpx.http_handler import HTTPHandler - - -class CustomLoggingHandler(CustomLogger): - response_cost: Optional[float] = None - - def __init__(self): - super().__init__() - - def log_success_event(self, kwargs, response_obj, start_time, end_time): - self.response_cost = kwargs["response_cost"] - - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - print(f"kwargs - {kwargs}") - print(f"kwargs response cost - {kwargs.get('response_cost')}") - self.response_cost = kwargs["response_cost"] - - print(f"response_cost: {self.response_cost} ") - - def log_failure_event(self, kwargs, response_obj, start_time, end_time): - print("Reaches log failure event!") - self.response_cost = kwargs["response_cost"] - - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): - print("Reaches async log failure event!") - self.response_cost = kwargs["response_cost"] - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.asyncio -async def test_custom_pricing(sync_mode): - new_handler = CustomLoggingHandler() - litellm.callbacks = [new_handler] - if sync_mode: - response = litellm.completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hey!"}], - mock_response="What do you want?", - input_cost_per_token=0.0, - output_cost_per_token=0.0, - ) - time.sleep(5) - else: - response = await litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hey!"}], - mock_response="What do you want?", - input_cost_per_token=0.0, - output_cost_per_token=0.0, - ) - - await asyncio.sleep(5) - - print(f"new_handler.response_cost: {new_handler.response_cost}") - assert new_handler.response_cost is not None - - assert new_handler.response_cost == 0 - - -@pytest.mark.parametrize( - "sync_mode", - [True, False], -) -@pytest.mark.asyncio -async def test_failure_completion_cost(sync_mode): - new_handler = CustomLoggingHandler() - litellm.callbacks = [new_handler] - if sync_mode: - try: - response = litellm.completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hey!"}], - mock_response=Exception("this should trigger an error"), - ) - except Exception: - pass - time.sleep(5) - else: - try: - response = await litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hey!"}], - mock_response=Exception("this should trigger an error"), - ) - except Exception: - pass - await asyncio.sleep(5) - - print(f"new_handler.response_cost: {new_handler.response_cost}") - assert new_handler.response_cost is not None - - assert new_handler.response_cost == 0 - - - - - # print(results) - - -# test_get_gpt3_tokens() - - -# test_get_palm_tokens() - - -# test_zephyr_hf_tokens() - - - - -# test_cost_ft_gpt_35() - - - - -# test_cost_azure_gpt_35() - - -# test_cost_azure_embedding() - - - - - - - - - - - - - - - - - - - - - - - - - - -# def test_vertex_ai_embedding_completion_cost_e2e(): -# """ -# Relevant issue - https://github.com/BerriAI/litellm/issues/4630 -# """ -# from test_amazing_vertex_completion import load_vertex_ai_credentials - -# load_vertex_ai_credentials() -# os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" -# litellm.model_cost = litellm.get_model_cost_map(url="") - -# text = "The quick brown fox jumps over the lazy dog." -# input_tokens = litellm.token_counter( -# model="vertex_ai/textembedding-gecko", text=text -# ) - -# model_info = litellm.get_model_info(model="vertex_ai/textembedding-gecko") - -# print("\nExpected model info:\n{}\n\n".format(model_info)) - -# expected_input_cost = input_tokens * model_info["input_cost_per_token"] - -# ## CALCULATED COST -# resp = litellm.embedding(model="textembedding-gecko", input=[text]) - -# calculated_input_cost = resp._hidden_params["response_cost"] - -# assert round(expected_input_cost, 6) == round(calculated_input_cost, 6) -# print("expected_input_cost: {}".format(expected_input_cost)) -# print("calculated_input_cost: {}".format(calculated_input_cost)) - -# assert False - - - - - - -def test_vertex_ai_llama_predict_cost(): - model = "meta/llama3-405b-instruct-maas" - messages = [{"role": "user", "content": "Hey, hows it going???"}] - custom_llm_provider = "vertex_ai" - predictive_cost = completion_cost( - model=model, messages=messages, custom_llm_provider=custom_llm_provider - ) - - assert predictive_cost == 0 - - - - - - - - -def test_completion_cost_azure_common_deployment_name(): - from litellm.utils import ( - CallTypes, - Choices, - Message, - ModelResponse, - Usage, - ) - - router = litellm.Router( - model_list=[ - { - "model_name": "gpt-4", - "litellm_params": { - "model": "azure/gpt-4-0314", - "max_tokens": 4096, - "api_key": os.getenv("AZURE_API_KEY"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - "model_info": {"base_model": "azure/gpt-4"}, - } - ] - ) - - response = ModelResponse( - id="chatcmpl-876cce24-e520-4cf8-8649-562a9be11c02", - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message( - content="Hi! I'm an AI, so I don't have emotions or feelings like humans do, but I'm functioning properly and ready to help with any questions or topics you'd like to discuss! How can I assist you today?", - role="assistant", - ), - ) - ], - created=1717519830, - model="gpt-4", - object="chat.completion", - system_fingerprint="fp_c1a4bcec29", - usage=Usage(completion_tokens=46, prompt_tokens=17, total_tokens=63), - ) - response._hidden_params["custom_llm_provider"] = "azure" - print(response) - - with patch.object( - litellm.cost_calculator, "completion_cost", new=MagicMock() - ) as mock_client: - _ = litellm.response_cost_calculator( - response_object=response, - model="gpt-4-0314", - custom_llm_provider="azure", - call_type=CallTypes.acompletion.value, - optional_params={}, - base_model="azure/gpt-4", - ) - - mock_client.assert_called() - - print(f"mock_client.call_args: {mock_client.call_args.kwargs}") - assert "azure/gpt-4" == mock_client.call_args.kwargs["base_model"] - - - - - - - - - - - - -@pytest.mark.parametrize( - "model", - [ - "fireworks_ai/accounts/fireworks/models/deepseek-v3p1", - ], -) -def test_completion_cost_fireworks_ai(model): - """ - Mocked so it does not depend on Fireworks' rotating serverless catalog. - Validates the Fireworks cost path: a parsed response with usage yields a - non-zero cost against the local cost map. - """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - mock_response_data = { - "id": "chatcmpl-test", - "object": "chat.completion", - "created": 1234567890, - "model": model.split("fireworks_ai/")[-1], - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "Going great, thanks!"}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 8, "completion_tokens": 5, "total_tokens": 13}, - } - - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"content-type": "application/json"} - mock_response.json.return_value = mock_response_data - mock_response.text = json.dumps(mock_response_data) - - sync_handler = HTTPHandler() - messages = [{"role": "user", "content": "Hey, how's it going?"}] - - with patch.object(HTTPHandler, "post", return_value=mock_response): - resp = litellm.completion(model=model, messages=messages, client=sync_handler) - - cost = completion_cost(completion_response=resp) - assert cost > 0 - - -def test_completion_cost_vertex_llama3(): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - from litellm.utils import Choices, Message, ModelResponse, Usage - - response = ModelResponse( - id="2024-09-19|14:52:01.823070-07|3.10.13.64|-333502972", - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message( - content="My name is Litellm Bot, and I'm here to help you with any questions or tasks you may have. As for the weather, I'd be happy to provide you with the current conditions and forecast for your location. However, I'm a large language model, I don't have real-time access to your location, so I'll need you to tell me where you are or provide me with a specific location you're interested in knowing the weather for.\\n\\nOnce you provide me with that information, I can give you the current weather conditions, including temperature, humidity, wind speed, and more, as well as a forecast for the next few days. Just let me know how I can assist you!", - role="assistant", - tool_calls=None, - function_call=None, - ), - ) - ], - created=1726782721, - model="vertex_ai/meta/llama3-405b-instruct-maas", - object="chat.completion", - system_fingerprint="", - usage=Usage( - completion_tokens=152, - prompt_tokens=27, - total_tokens=179, - completion_tokens_details=None, - ), - ) - - model = "vertex_ai/meta/llama3-8b-instruct-maas" - cost = completion_cost(model=model, completion_response=response) - - assert cost == 0 - - - - - - - - - - - - - - - - - - -def test_completion_cost_azure_tts(): - from unittest.mock import MagicMock - - args = { - "response_object": MagicMock, - "model": "tts-1", - "cache_hit": None, - "custom_llm_provider": "azure", - "base_model": None, - "call_type": "aspeech", - "optional_params": {}, - "custom_pricing": False, - } - litellm.response_cost_calculator(**args) - - - - -def test_moderations(): - from litellm import moderation - - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - litellm.add_known_models() - - assert "omni-moderation-latest" in litellm.model_cost - print( - f"litellm.model_cost['omni-moderation-latest']: {litellm.model_cost['omni-moderation-latest']}" - ) - assert "omni-moderation-latest" in litellm.open_ai_chat_completion_models - - response = moderation("I am a bad person", model="omni-moderation-latest") - cost = completion_cost(response, model="omni-moderation-latest") - assert cost == 0 - - -def test_cost_calculator_azure_embedding(): - from litellm.cost_calculator import response_cost_calculator - from litellm.types.utils import EmbeddingResponse, Usage - - kwargs = { - "response_object": EmbeddingResponse( - model="text-embedding-3-small", - data=[{"embedding": [1, 2, 3]}], - usage=Usage(prompt_tokens=10, completion_tokens=10), - ), - "model": "text-embedding-3-small", - "cache_hit": None, - "custom_llm_provider": None, - "base_model": "azure/text-embedding-3-small", - "call_type": "aembedding", - "optional_params": {}, - "custom_pricing": False, - "prompt": "Hello, world!", - } - - try: - response_cost_calculator(**kwargs) - except Exception as e: - traceback.print_exc() - pytest.fail(f"Error: {e}") - - -def test_add_known_models(): - litellm.add_known_models() - assert ( - "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0" not in litellm.bedrock_models - ) - - - - -# @pytest.mark.parametrize( -# "base_model_arg", [ -# {"base_model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0"}, -# {"model_info": "anthropic.claude-3-sonnet-20240229-v1:0"}, -# ] -# ) diff --git a/tests/local_testing/test_cost_calc.py b/tests/local_testing/test_cost_calc.py deleted file mode 100644 index 0b2e8e39701..00000000000 --- a/tests/local_testing/test_cost_calc.py +++ /dev/null @@ -1,115 +0,0 @@ -import traceback - -from dotenv import load_dotenv - -load_dotenv() -import io - -from typing import Literal - -import pytest -from pydantic import BaseModel, ConfigDict - -import litellm -from litellm import Router, completion_cost, stream_chunk_builder - -models = [ - dict( - model_name="openai/gpt-3.5-turbo", - ), - dict( - model_name="anthropic/claude-3-haiku-20240307", - ), - dict( - model_name="together_ai/meta-llama/Llama-2-7b-chat-hf", - ), -] - -router = Router( - model_list=[ - { - "model_name": m["model_name"], - "litellm_params": { - "model": m.get("model", m["model_name"]), - }, - } - for m in models - ], - routing_strategy="simple-shuffle", - num_retries=3, - retry_after=1, - timeout=60.0, - allowed_fails=2, - cooldown_time=0, - debug_level="INFO", -) - - -@pytest.mark.parametrize( - "model", - [ - "openai/gpt-3.5-turbo", - # "anthropic/claude-3-haiku-20240307", - # "together_ai/meta-llama/Llama-2-7b-chat-hf", - ], -) -def test_run(model: str): - """ - Relevant issue - https://github.com/BerriAI/litellm/issues/4965 - """ - litellm.set_verbose = True - prompt = "Hi" - kwargs = dict( - model=model, - messages=[{"role": "user", "content": prompt}], - temperature=0.001, - top_p=0.001, - max_tokens=20, - input_cost_per_token=2, - output_cost_per_token=2, - ) - - print(f"--------- {model} ---------") - print(f"Prompt: {prompt}") - - response = router.completion(**kwargs) # type: ignore - non_stream_output = response.choices[0].message.content.replace("\n", "") # type: ignore - non_stream_cost_calc = response._hidden_params["response_cost"] * 100 - - print(f"Non-stream output: {non_stream_output}") - print(f"Non-stream usage : {response.usage}") # type: ignore - non_stream_usage = response.usage - try: - print( - f"Non-stream cost : {response._hidden_params['response_cost'] * 100:.4f}" - ) - except TypeError: - print("Non-stream cost : NONE") - print(f"Non-stream cost : {completion_cost(response) * 100:.4f} (response)") - - response = router.completion(**kwargs, stream=True, stream_options={"include_usage": True}) # type: ignore - response = stream_chunk_builder(list(response), messages=kwargs["messages"]) # type: ignore - output = response.choices[0].message.content.replace("\n", "") # type: ignore - - if response.usage.completion_tokens != non_stream_usage.completion_tokens: - pytest.skip( - "LLM API returning inconsistent usage" - ) # handles transient openai errors - streaming_cost_calc = ( - completion_cost( - response, - custom_cost_per_token={ - "input_cost_per_token": kwargs["input_cost_per_token"], - "output_cost_per_token": kwargs["output_cost_per_token"], - }, - ) - * 100 - ) - print(f"Stream output : {output}") - - print(f"Stream usage : {response.usage}") # type: ignore - print(f"Stream cost : {streaming_cost_calc} (response)") - print("") - if output == non_stream_output: - # assert cost is the same - assert streaming_cost_calc == non_stream_cost_calc diff --git a/tests/local_testing/test_docker_no_network_on_deploy.py b/tests/local_testing/test_docker_no_network_on_deploy.py deleted file mode 100644 index b73b103eb7e..00000000000 --- a/tests/local_testing/test_docker_no_network_on_deploy.py +++ /dev/null @@ -1,385 +0,0 @@ -""" -Test to ensure Docker container does not go out to network on deploy. - -This test verifies that the LiteLLM proxy container does not make outbound -network requests during startup. This is important for: -1. Air-gapped environments where outbound network is restricted -2. Security compliance requiring no unexpected network calls -3. Fast container startup without network dependencies - -The test works by: -1. Building/running the container with network disabled -2. Verifying the container starts successfully without network -3. Checking for any errors related to network failures during startup -""" - -import os -import re -import subprocess -import time - -import pytest - -from tests._master_key import MASTER_KEY - - -def is_docker_available() -> bool: - """Check if Docker is available and running.""" - try: - result = subprocess.run( - ["docker", "info"], - capture_output=True, - text=True, - timeout=10, - ) - return result.returncode == 0 - except (subprocess.TimeoutExpired, FileNotFoundError): - return False - - -@pytest.mark.skipif( - not is_docker_available(), - reason="Docker not available", -) -class TestDockerNoNetworkOnDeploy: - """ - Test suite for verifying Docker container starts without network access. - """ - - # Container and image names for testing - TEST_IMAGE_NAME = "litellm-no-network-test" - TEST_CONTAINER_NAME = "litellm-no-network-test-container" - - # Timeout for container operations - CONTAINER_START_TIMEOUT = 60 # seconds - - # Patterns that indicate network-related failures during startup - NETWORK_ERROR_PATTERNS = [ - r"connection refused", - r"network is unreachable", - r"could not resolve host", - r"name resolution failed", - r"dns lookup failed", - r"failed to establish.*connection", - r"socket.gaierror", - r"urllib.error.URLError.*Errno", - r"requests.exceptions.ConnectionError", - r"httpx.*ConnectError", - r"aiohttp.*ClientConnectorError", - ] - - # Patterns that indicate EXPECTED local-only startup - LOCAL_STARTUP_PATTERNS = [ - r"starting.*(server|proxy)", - r"listening on", - r"uvicorn running", - r"application startup complete", - ] - - @pytest.fixture(autouse=True) - def cleanup(self): - """Cleanup any existing test containers before and after each test.""" - self._cleanup_container() - yield - self._cleanup_container() - - def _cleanup_container(self): - """Remove test container if it exists.""" - subprocess.run( - ["docker", "rm", "-f", self.TEST_CONTAINER_NAME], - capture_output=True, - timeout=30, - ) - - def _build_test_image(self) -> bool: - """ - Build the Docker image if needed. - Returns True if image is available (built or already exists). - """ - # Check if main litellm image exists - result = subprocess.run( - ["docker", "images", "-q", "litellm/litellm"], - capture_output=True, - text=True, - timeout=10, - ) - if result.stdout.strip(): - # Image exists, use it - return True - - # Try to build from Dockerfile - dockerfile_path = os.path.join( - os.path.dirname(__file__), - "..", - "..", - "..", - "Dockerfile", - ) - if os.path.exists(dockerfile_path): - result = subprocess.run( - [ - "docker", - "build", - "-t", - self.TEST_IMAGE_NAME, - "-f", - dockerfile_path, - os.path.dirname(dockerfile_path), - ], - capture_output=True, - text=True, - timeout=600, # 10 minutes for build - ) - return result.returncode == 0 - - return False - - def test_container_starts_without_network(self): - """ - Test that the container can start with network completely disabled. - - This test runs the container with --network=none to ensure no outbound - network requests are required during startup. - """ - # Use a minimal config that doesn't require external services - minimal_config = f""" -model_list: - - model_name: fake-model - litellm_params: - model: fake/fake-model - -general_settings: - master_key: {MASTER_KEY} - database_url: null - -environment_variables: {{}} -""" - - # Create a temporary config file - config_path = "/tmp/litellm_test_config.yaml" - with open(config_path, "w") as f: - f.write(minimal_config) - - image_to_use = "litellm/litellm" - # Check if image exists - result = subprocess.run( - ["docker", "images", "-q", image_to_use], - capture_output=True, - text=True, - timeout=10, - ) - if not result.stdout.strip(): - pytest.skip(f"Docker image {image_to_use} not available") - - # Run container with network disabled - run_cmd = [ - "docker", - "run", - "--name", - self.TEST_CONTAINER_NAME, - "--network=none", # Disable all network access - "-v", - f"{config_path}:/app/config.yaml:ro", - "-e", - f"LITELLM_MASTER_KEY={MASTER_KEY}", - "-e", - "DATABASE_URL=", # Empty to disable DB - "-e", - "STORE_MODEL_IN_DB=false", - "-e", - "LITELLM_LOG=DEBUG", - "-d", # Detached mode - image_to_use, - "--config", - "/app/config.yaml", - ] - - result = subprocess.run( - run_cmd, - capture_output=True, - text=True, - timeout=30, - ) - - if result.returncode != 0: - pytest.fail(f"Failed to start container: {result.stderr}") - - # Wait for container to start up - time.sleep(5) - - # Check container logs for any network-related errors - logs_result = subprocess.run( - ["docker", "logs", self.TEST_CONTAINER_NAME], - capture_output=True, - text=True, - timeout=30, - ) - - logs = logs_result.stdout + logs_result.stderr - - # Check for network error patterns (case-insensitive) - network_errors = [] - for pattern in self.NETWORK_ERROR_PATTERNS: - matches = re.findall(pattern, logs, re.IGNORECASE) - if matches: - network_errors.extend(matches) - - # Check if container is still running - inspect_result = subprocess.run( - [ - "docker", - "inspect", - "-f", - "{{.State.Running}}", - self.TEST_CONTAINER_NAME, - ], - capture_output=True, - text=True, - timeout=10, - ) - - is_running = inspect_result.stdout.strip() == "true" - - # If container crashed, get exit code and reason - - # If container crashed, get exit code and reason - if not is_running: - exit_result = subprocess.run( - [ - "docker", - "inspect", - "-f", - "{{.State.ExitCode}}", - self.TEST_CONTAINER_NAME, - ], - capture_output=True, - text=True, - timeout=10, - ) - exit_code = exit_result.stdout.strip() - - # Container not running is OK if it didn't crash due to network issues - # Check if exit was due to network errors - if network_errors: - pytest.fail( - f"Container failed with network errors (exit code {exit_code}): " - f"{network_errors}\n\nFull logs:\n{logs}" - ) - - # Assert no network errors were found - assert len(network_errors) == 0, ( - f"Container made network requests during startup that failed: " - f"{network_errors}" - ) - - def test_no_external_urls_in_startup_code(self): - """ - Static analysis test: check that startup code doesn't contain - hardcoded external URLs that would be called during import/startup. - - This is a complementary test to catch issues without needing Docker. - """ - # Directories to check for startup code - startup_dirs = [ - "litellm/proxy", - "litellm/__init__.py", - "litellm/main.py", - ] - - # Patterns that indicate external URL calls during startup (not in functions) - problematic_patterns = [ - # Immediate HTTP calls (not inside functions) - r'^requests\.get\(["\'](https?://)', - r'^httpx\.get\(["\'](https?://)', - r'^urllib\.request\.urlopen\(["\'](https?://)', - ] - - # Files that are OK to have URLs (they're called on-demand, not startup) - allowed_files = [ - "model_prices_and_context_window.json", # Static data file - "test_", # Test files - "_test.py", - "conftest.py", - ] - - workspace_root = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) - - issues_found = [] - - for startup_dir in startup_dirs: - full_path = os.path.join(workspace_root, startup_dir) - if not os.path.exists(full_path): - continue - - if os.path.isfile(full_path): - files_to_check = [full_path] - else: - files_to_check = [] - for root, _dirs, files in os.walk(full_path): - for f in files: - if f.endswith(".py"): - files_to_check.append(os.path.join(root, f)) - - for filepath in files_to_check: - # Skip allowed files - if any(allowed in filepath for allowed in allowed_files): - continue - - try: - with open(filepath, "r") as f: - content = f.read() - - for pattern in problematic_patterns: - matches = re.findall(pattern, content, re.MULTILINE) - if matches: - issues_found.append(f"{filepath}: {pattern} matched") - except Exception: - pass # Skip unreadable files - - # This test is informational - we document but don't fail - if issues_found: - pass - - -@pytest.mark.skipif( - not is_docker_available(), - reason="Docker not available", -) -def test_container_build_no_network_fetch(): - """ - Test that the Docker build process doesn't require network for runtime. - - This verifies that all dependencies are properly bundled and no - runtime network calls are made during container initialization. - - Note: Build itself may need network to resolve dependencies, but runtime should not. - """ - # This is a simplified version - full test would need to: - # 1. Build image with --network=none (requires pre-cached deps) - # 2. Or run built image in isolated network - - # For now, just verify the Dockerfile doesn't have wget/curl in CMD/ENTRYPOINT - workspace_root = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) - dockerfile_path = os.path.join(workspace_root, "Dockerfile") - - if not os.path.exists(dockerfile_path): - pytest.skip("Dockerfile not found") - - with open(dockerfile_path, "r") as f: - content = f.read() - - # Check for network calls in CMD/ENTRYPOINT - problematic = [] - lines = content.split("\n") - for i, line in enumerate(lines, 1): - line_upper = line.strip().upper() - if line_upper.startswith(("CMD", "ENTRYPOINT")): - if any( - cmd in line.lower() - for cmd in ["curl", "wget", "fetch", "http://", "https://"] - ): - problematic.append(f"Line {i}: {line.strip()}") - - assert ( - len(problematic) == 0 - ), f"Dockerfile CMD/ENTRYPOINT contains network calls: {problematic}" diff --git a/tests/local_testing/test_embedding.py b/tests/local_testing/test_embedding.py deleted file mode 100644 index 83ebfa102a3..00000000000 --- a/tests/local_testing/test_embedding.py +++ /dev/null @@ -1,1224 +0,0 @@ -import json -import os - -import httpx -import openai -import pytest -from dotenv import load_dotenv - -load_dotenv() - -from unittest.mock import MagicMock, patch - -from openai.types import CreateEmbeddingResponse -from openai.types.create_embedding_response import Usage as EmbeddingUsage - -import litellm -from litellm import completion_cost, embedding -from tests.capturing_transport import CapturingTransport -from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE - -litellm.set_verbose = False - - -def test_openai_embedding(): - try: - litellm.set_verbose = True - response = embedding( - model="text-embedding-ada-002", - input=["good morning from litellm", "this is another item"], - metadata={"anything": "good day"}, - ) - litellm_response = dict(response) - litellm_response_keys = set(litellm_response.keys()) - litellm_response_keys.discard("_response_ms") - - print(litellm_response_keys) - print("LiteLLM Response\n") - # print(litellm_response) - - # same request with OpenAI 1.0+ - import openai - - client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"]) - response = client.embeddings.create( - model="text-embedding-ada-002", - input=["good morning from litellm", "this is another item"], - ) - - response = dict(response) - openai_response_keys = set(response.keys()) - print(openai_response_keys) - assert ( - litellm_response_keys == openai_response_keys - ) # ENSURE the Keys in litellm response is exactly what the openai package returns - assert ( - len(litellm_response["data"]) == 2 - ) # expect two embedding responses from litellm_response since input had two - print(openai_response_keys) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_openai_embedding() - - -def test_openai_embedding_3(): - try: - litellm.set_verbose = True - response = embedding( - model="text-embedding-3-small", - input=["good morning from litellm", "this is another item"], - metadata={"anything": "good day"}, - dimensions=5, - ) - print(f"response:", response) - litellm_response = dict(response) - litellm_response_keys = set(litellm_response.keys()) - litellm_response_keys.discard("_response_ms") - - print(litellm_response_keys) - print("LiteLLM Response\n") - # print(litellm_response) - - # same request with OpenAI 1.0+ - import openai - - client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"]) - response = client.embeddings.create( - model="text-embedding-3-small", - input=["good morning from litellm", "this is another item"], - dimensions=5, - ) - - response = dict(response) - openai_response_keys = set(response.keys()) - print(openai_response_keys) - assert ( - litellm_response_keys == openai_response_keys - ) # ENSURE the Keys in litellm response is exactly what the openai package returns - assert ( - len(litellm_response["data"]) == 2 - ) # expect two embedding responses from litellm_response since input had two - print(openai_response_keys) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -@pytest.mark.parametrize( - "model, api_base, api_key", - [ - # ("azure/text-embedding-ada-002", None, None), - ( - "together_ai/BAAI/bge-base-en-v1.5", - None, - None, - ), # Updated to current Together AI embedding model - ], -) -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.asyncio -async def test_together_ai_embedding(model, api_base, api_key, sync_mode): - try: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - # litellm.set_verbose = True - if sync_mode: - response = embedding( - model=model, - input=["good morning from litellm"], - api_base=api_base, - api_key=api_key, - ) - else: - response = await litellm.aembedding( - model=model, - input=["good morning from litellm"], - api_base=api_base, - api_key=api_key, - ) - # print(await response) - print(response) - print(response._hidden_params) - response_keys = set(dict(response).keys()) - response_keys.discard("_response_ms") - assert set(["usage", "model", "object", "data"]) == set( - response_keys - ) # assert litellm response has expected keys from OpenAI embedding response - - request_cost = litellm.completion_cost( - completion_response=response, call_type="embedding" - ) - - print("Calculated request cost=", request_cost) - - assert isinstance(response.usage, litellm.Usage) - except litellm.BadRequestError: - print( - "Bad request error occurred - Together AI raises 404s for their embedding models" - ) - pass - - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_openai_azure_embedding_simple() -import base64 - -import requests - -litellm.set_verbose = True -url = "https://dummyimage.com/100/100/fff&text=Test+image" -response = requests.get(url) -file_data = response.content - -encoded_file = base64.b64encode(file_data).decode("utf-8") -base64_image = f"data:image/png;base64,{encoded_file}" - - -from openai.types.embedding import Embedding - - -def _azure_ai_image_mock_response(*args, **kwargs): - new_response = MagicMock() - new_response.headers = {"azureml-model-group": "offer-cohere-embed-multili-paygo"} - - new_response.json.return_value = { - "data": [Embedding(embedding=[1234], index=0, object="embedding")], - "model": "", - "object": "list", - "usage": {"prompt_tokens": 1, "total_tokens": 2}, - } - - return new_response - - -@pytest.mark.parametrize("sync_mode", [True]) # , False -@pytest.mark.asyncio -async def test_azure_ai_embedding_image(sync_mode): - model = "azure_ai/Cohere-embed-v3-multilingual-2" - api_base = os.getenv("AZURE_AI_API_BASE") - api_key = os.getenv("AZURE_AI_API_KEY") - try: - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - input = base64_image - if sync_mode: - client = HTTPHandler() - else: - client = AsyncHTTPHandler() - with patch.object( - client, "post", side_effect=_azure_ai_image_mock_response - ) as mock_client: - if sync_mode: - response = embedding( - model=model, - input=[input], - api_base=api_base, - api_key=api_key, - client=client, - ) - else: - response = await litellm.aembedding( - model=model, - input=[input], - api_base=api_base, - api_key=api_key, - client=client, - ) - print(response) - - assert len(response.data) == 1 - - print(response._hidden_params) - response_keys = set(dict(response).keys()) - response_keys.discard("_response_ms") - assert set(["usage", "model", "object", "data"]) == set( - response_keys - ) # assert litellm response has expected keys from OpenAI embedding response - - request_cost = litellm.completion_cost(completion_response=response) - - print("Calculated request cost=", request_cost) - - assert isinstance(response.usage, litellm.Usage) - - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -def test_openai_azure_embedding_timeouts(): - try: - response = embedding( - model="azure/text-embedding-ada-002", - input=["good morning from litellm"], - timeout=0.00001, - ) - print(response) - except openai.APITimeoutError: - print("Good job got timeout error!") - pass - except Exception as e: - pytest.fail( - f"Expected timeout error, did not get the correct error. Instead got {e}" - ) - - -# test_openai_azure_embedding_timeouts() - - -def test_openai_embedding_timeouts(): - try: - response = embedding( - model="openai/slow-endpoint", - input=["good morning from litellm"], - api_base=FAKE_OPENAI_API_BASE, - api_key="fake-key", - timeout=0.5, - ) - print(response) - pytest.fail("Expected timeout error, the request returned instead") - except openai.APITimeoutError: - print("Good job got OpenAI timeout error!") - pass - except Exception as e: - pytest.fail( - f"Expected timeout error, did not get the correct error. Instead got {e}" - ) - - -# test_openai_embedding_timeouts() - - -def test_openai_azure_embedding(): - try: - api_key = os.environ["AZURE_AI_API_KEY"] - api_base = os.environ["AZURE_AI_API_BASE"] - api_version = os.environ["AZURE_API_VERSION"] - - os.environ["AZURE_API_VERSION"] = "" - os.environ["AZURE_AI_API_BASE"] = "" - os.environ["AZURE_AI_API_KEY"] = "" - - response = embedding( - model="azure/text-embedding-ada-002", - input=["good morning from litellm", "this is another item"], - api_key=api_key, - api_base=api_base, - api_version=api_version, - ) - print(response) - - os.environ["AZURE_API_VERSION"] = api_version - os.environ["AZURE_AI_API_BASE"] = api_base - os.environ["AZURE_AI_API_KEY"] = api_key - - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - - - -def _openai_mock_response(*args, **kwargs): - new_response = MagicMock() - new_response.headers = {"hello": "world"} - - new_response.parse.return_value = ( - openai.types.create_embedding_response.CreateEmbeddingResponse( - data=[Embedding(embedding=[1234, 45667], index=0, object="embedding")], - model="azure/test", - object="list", - usage=openai.types.create_embedding_response.Usage( - prompt_tokens=1, total_tokens=2 - ), - ) - ) - return new_response - - -def test_openai_azure_embedding_optional_arg(): - - with patch.object( - openai.resources.embeddings.Embeddings, - "create", - side_effect=_openai_mock_response, - ) as mock_client: - _ = litellm.embedding( - model="azure/test", - input=["test"], - api_version="test", - api_base="test", - azure_ad_token="test", - ) - - mock_client.assert_called_once_with( - model="test", - input=["test"], - extra_body={"azure_ad_token": "test"}, - timeout=600, - extra_headers={"X-Stainless-Raw-Response": "true"}, - ) - # Verify azure_ad_token is passed in extra_body, not as a direct parameter - assert "azure_ad_token" not in mock_client.call_args.kwargs - assert mock_client.call_args.kwargs["extra_body"]["azure_ad_token"] == "test" - - -# test_openai_azure_embedding() - -# test_openai_embedding() - - -# test_cohere_embedding() - - -@pytest.mark.parametrize("custom_llm_provider", ["cohere", "cohere_chat"]) -@pytest.mark.asyncio() -async def test_cohere_embedding3(custom_llm_provider): - try: - litellm.set_verbose = True - response = await litellm.aembedding( - model=f"{custom_llm_provider}/embed-english-v3.0", - input=["good morning from litellm", "this is another item"], - timeout=None, - max_retries=0, - ) - print(f"response:", response) - - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_cohere_embedding3() - - -@pytest.mark.parametrize( - "model", - [ - "bedrock/amazon.titan-embed-text-v1", - "bedrock/amazon.titan-embed-image-v1", - "bedrock/amazon.titan-embed-text-v2:0", - ], -) -@pytest.mark.parametrize("sync_mode", [True, False]) # , -@pytest.mark.asyncio -async def test_bedrock_embedding_titan(model, sync_mode): - try: - # this tests if we support str input for bedrock embedding - litellm.set_verbose = True - litellm.enable_cache() - import time - - current_time = str(time.time()) - # DO NOT MAKE THE INPUT A LIST in this test - if sync_mode: - response = embedding( - model=model, - input=f"good morning from litellm, attempting to embed data {current_time}", # input should always be a string in this test - aws_region_name="us-west-2", - ) - else: - response = await litellm.aembedding( - model=model, - input=f"good morning from litellm, attempting to embed data {current_time}", # input should always be a string in this test - aws_region_name="us-west-2", - ) - print("response:", response) - assert isinstance( - response["data"][0]["embedding"], list - ), "Expected response to be a list" - print("type of first embedding:", type(response["data"][0]["embedding"][0])) - assert all( - isinstance(x, float) for x in response["data"][0]["embedding"] - ), "Expected response to be a list of floats" - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -@pytest.mark.parametrize( - "model", - [ - "bedrock/amazon.titan-embed-text-v1", - "bedrock/amazon.titan-embed-image-v1", - "bedrock/amazon.titan-embed-text-v2:0", - ], -) -@pytest.mark.parametrize("sync_mode", [True]) # True, -@pytest.mark.asyncio -async def test_bedrock_embedding_titan_caching(model, sync_mode): - try: - # this tests if we support str input for bedrock embedding - litellm.set_verbose = True - litellm.enable_cache() - import time - - current_time = str(time.time()) - # DO NOT MAKE THE INPUT A LIST in this test - if sync_mode: - response = embedding( - model=model, - input=f"good morning from litellm, attempting to embed data {current_time}", # input should always be a string in this test - aws_region_name="us-west-2", - ) - else: - response = await litellm.aembedding( - model=model, - input=f"good morning from litellm, attempting to embed data {current_time}", # input should always be a string in this test - aws_region_name="us-west-2", - ) - print("response:", response) - assert isinstance( - response["data"][0]["embedding"], list - ), "Expected response to be a list" - print("type of first embedding:", type(response["data"][0]["embedding"][0])) - assert all( - isinstance(x, float) for x in response["data"][0]["embedding"] - ), "Expected response to be a list of floats" - - # this also tests if we can return a cache response for this scenario - import time - - start_time = time.time() - - if sync_mode: - response = embedding( - model=model, - input=f"good morning from litellm, attempting to embed data {current_time}", # input should always be a string in this test - ) - else: - response = await litellm.aembedding( - model=model, - input=f"good morning from litellm, attempting to embed data {current_time}", # input should always be a string in this test - ) - print(response) - - end_time = time.time() - print(response._hidden_params) - print(f"Embedding 2 response time: {end_time - start_time} seconds") - - assert end_time - start_time < 0.1 - litellm.disable_cache() - - assert isinstance(response.usage, litellm.Usage) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_bedrock_embedding_titan() - - -def test_bedrock_embedding_cohere(): - try: - litellm.set_verbose = False - response = embedding( - model="cohere.embed-multilingual-v3", - input=[ - "good morning from litellm, attempting to embed data", - "lets test a second string for good measure", - ], - aws_region_name="us-west-2", - ) - assert isinstance( - response["data"][0]["embedding"], list - ), "Expected response to be a list" - print(f"type of first embedding:", type(response["data"][0]["embedding"][0])) - assert all( - isinstance(x, float) for x in response["data"][0]["embedding"] - ), "Expected response to be a list of floats" - # print(f"response:", response) - - assert isinstance(response.usage, litellm.Usage) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_bedrock_embedding_cohere() - - -# comment out hf tests - since hf endpoints are unstable -def test_hf_embedding(): - try: - # huggingface/microsoft/codebert-base - # huggingface/facebook/bart-large - response = embedding( - model="huggingface/sentence-transformers/all-MiniLM-L6-v2", - input=["good morning from litellm", "this is another item"], - ) - print(f"response:", response) - - assert isinstance(response.usage, litellm.Usage) - except Exception as e: - # Note: Huggingface inference API is unstable and fails with "model loading errors all the time" - pass - - -# test_hf_embedding() - - - -def tgi_mock_post(*args, **kwargs): - import json - - expected_data = { - "inputs": { - "source_sentence": "good morning from litellm", - "sentences": ["this is another item"], - } - } - assert ( - json.loads(kwargs["data"]) == expected_data - ), "Data does not match the expected data" - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - mock_response.json.return_value = [0.7708950042724609] - return mock_response - - -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler - - -@pytest.mark.asyncio -@patch( - "litellm.llms.huggingface.embedding.handler.async_get_hf_task_embedding_for_model" -) -@patch("litellm.llms.huggingface.embedding.handler.get_hf_task_embedding_for_model") -@pytest.mark.parametrize("sync_mode", [True, False]) -async def test_hf_embedding_sentence_sim( - mock_async_get_hf_task_embedding_for_model, - mock_get_hf_task_embedding_for_model, - sync_mode, -): - try: - # huggingface/microsoft/codebert-base - # huggingface/facebook/bart-large - mock_get_hf_task_embedding_for_model.return_value = "sentence-similarity" - mock_async_get_hf_task_embedding_for_model.return_value = "sentence-similarity" - if sync_mode is True: - client = HTTPHandler(concurrent_limit=1) - else: - client = AsyncHTTPHandler(concurrent_limit=1) - with patch.object(client, "post", side_effect=tgi_mock_post) as mock_client: - data = { - "model": "huggingface/sentence-transformers/TaylorAI/bge-micro-v2", - "input": ["good morning from litellm", "this is another item"], - "client": client, - } - if sync_mode is True: - response = embedding(**data) - else: - response = await litellm.aembedding(**data) - - print(f"response:", response) - - mock_client.assert_called_once() - - assert isinstance(response.usage, litellm.Usage) - - except Exception as e: - # Note: Huggingface inference API is unstable and fails with "model loading errors all the time" - raise e - - -# test async embeddings -def test_aembedding(): - try: - import asyncio - - async def embedding_call(): - try: - response = await litellm.aembedding( - model="text-embedding-ada-002", - input=["good morning from litellm", "this is another item"], - ) - print(response) - return response - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - response = asyncio.run(embedding_call()) - print("Before caclulating cost, response", response) - - cost = litellm.completion_cost(completion_response=response) - - print("COST=", cost) - assert cost == float("1e-06") - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_aembedding() - - -def test_aembedding_azure(): - try: - import asyncio - - async def embedding_call(): - try: - response = await litellm.aembedding( - model="azure/text-embedding-ada-002", - input=["good morning from litellm", "this is another item"], - ) - print(response) - - print( - "hidden params - custom_llm_provider", - response._hidden_params["custom_llm_provider"], - ) - assert response._hidden_params["custom_llm_provider"] == "azure" - - assert isinstance(response.usage, litellm.Usage) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - asyncio.run(embedding_call()) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_aembedding_azure() - - - - - - -def test_mistral_embeddings(): - try: - litellm.set_verbose = True - response = litellm.embedding( - model="mistral/mistral-embed", - input=["good morning from litellm"], - ) - print(f"response: {response}") - assert isinstance(response.usage, litellm.Usage) - except litellm.RateLimitError as e: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -def test_fireworks_embeddings(): - try: - litellm.set_verbose = True - response = litellm.embedding( - model="fireworks_ai/nomic-ai/nomic-embed-text-v1.5", - input=["good morning from litellm"], - ) - print(f"response: {response}") - assert isinstance(response.usage, litellm.Usage) - cost = completion_cost(completion_response=response) - print("cost", cost) - assert cost > 0.0 - print(response._hidden_params) - assert response._hidden_params["response_cost"] > 0.0 - except litellm.RateLimitError as e: - pass - except litellm.InternalServerError as e: - pass - except litellm.APIError as e: - if "suspended" in str(e): - pytest.skip(f"Fireworks account suspended: {e}") - pytest.fail(f"Error occurred: {e}") - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -def test_watsonx_embeddings(monkeypatch): - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - # Mock the IAM token generation to avoid actual API calls - monkeypatch.setenv("WATSONX_API_KEY", "mock-api-key") - monkeypatch.setenv("WATSONX_TOKEN", "mock-watsonx-token") - monkeypatch.setenv("WATSONX_API_BASE", "https://us-south.ml.cloud.ibm.com") - monkeypatch.setenv("WATSONX_PROJECT_ID", "mock-project-id") - - client = HTTPHandler() - - # Track the actual request made - captured_request = {} - - def mock_wx_embed_request(url: str, **kwargs): - # Capture request details for verification - captured_request["url"] = url - captured_request["headers"] = kwargs.get("headers", {}) - captured_request["data"] = kwargs.get("data") - - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - mock_response.json.return_value = { - "model_id": "ibm/slate-30m-english-rtrvr", - "created_at": "2024-01-01T00:00:00.00Z", - "results": [{"embedding": [0.0] * 254}], - "input_token_count": 8, - } - return mock_response - - try: - litellm.set_verbose = True - with patch.object(client, "post", side_effect=mock_wx_embed_request): - response = litellm.embedding( - model="watsonx/ibm/slate-30m-english-rtrvr", - input=["good morning from litellm"], - client=client, - ) - - print(f"response: {response}") - assert isinstance(response.usage, litellm.Usage) - - # Verify the request was made correctly - assert "Authorization" in captured_request["headers"] - assert ( - captured_request["headers"]["Authorization"] == "Bearer mock-watsonx-token" - ) - assert "us-south.ml.cloud.ibm.com" in captured_request["url"] - except litellm.RateLimitError as e: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -@pytest.mark.asyncio -async def test_watsonx_aembeddings(monkeypatch): - from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler - - # Mock the IAM token generation to avoid actual API calls - monkeypatch.setenv("WATSONX_API_KEY", "mock-api-key") - monkeypatch.setenv("WATSONX_TOKEN", "mock-watsonx-token") - monkeypatch.setenv("WATSONX_API_BASE", "https://us-south.ml.cloud.ibm.com") - monkeypatch.setenv("WATSONX_PROJECT_ID", "mock-project-id") - - client = AsyncHTTPHandler() - - def mock_async_client(*args, **kwargs): - - mocked_client = MagicMock() - - async def mock_send(request, *args, stream: bool = False, **kwags): - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.headers = {"Content-Type": "application/json"} - mock_response.json.return_value = { - "model_id": "ibm/slate-30m-english-rtrvr", - "created_at": "2024-01-01T00:00:00.00Z", - "results": [{"embedding": [0.0] * 254}], - "input_token_count": 8, - } - mock_response.is_error = False - return mock_response - - mocked_client.send = mock_send - - return mocked_client - - try: - litellm.set_verbose = True - with patch.object(client, "post", side_effect=mock_async_client) as mock_client: - response = await litellm.aembedding( - model="watsonx/ibm/slate-30m-english-rtrvr", - input=["good morning from litellm"], - client=client, - ) - mock_client.assert_called_once() - print(f"response: {response}") - assert isinstance(response.usage, litellm.Usage) - except litellm.RateLimitError as e: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_mistral_embeddings() - - - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.parametrize( - "input", ["good morning from litellm", ["good morning from litellm"]] # -) -@pytest.mark.asyncio -async def test_gemini_embeddings(sync_mode, input): - try: - litellm.set_verbose = True - if sync_mode: - response = litellm.embedding( - model="gemini/gemini-embedding-001", - input=input, - ) - else: - response = await litellm.aembedding( - model="gemini/gemini-embedding-001", - input=input, - ) - print(f"response: {response}") - - # stubbed endpoint is setup to return this - assert isinstance(response.data[0]["embedding"], list) - assert response.usage.prompt_tokens > 0 - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_voyage_embeddings() -# def test_xinference_embeddings(): -# try: -# litellm.set_verbose = True -# response = litellm.embedding( -# model="xinference/bge-base-en", -# input=["good morning from litellm"], -# ) -# print(f"response: {response}") -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") -# test_xinference_embeddings() - -# test_sagemaker_embeddings() -# def local_proxy_embeddings(): -# litellm.set_verbose=True -# response = embedding( -# model="openai/custom_embedding", -# input=["good morning from litellm"], -# api_base="http://0.0.0.0:8000/" -# ) -# print(response) - -# local_proxy_embeddings() - - - - -def test_hosted_vllm_embedding(monkeypatch): - monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://localhost:8000") - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - client = HTTPHandler() - with patch.object(client, "post") as mock_post: - try: - embedding( - model="hosted_vllm/jina-embeddings-v3", - input=["Hello world"], - client=client, - ) - except Exception as e: - print(e) - - mock_post.assert_called_once() - - json_data = json.loads(mock_post.call_args.kwargs["data"]) - assert json_data["input"] == ["Hello world"] - assert json_data["model"] == "jina-embeddings-v3" - - -def test_llamafile_embedding(monkeypatch): - monkeypatch.setenv("LLAMAFILE_API_BASE", "http://localhost:8080/v1") - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - client = HTTPHandler() - with patch.object(client, "post") as mock_post: - try: - embedding( - model="llamafile/jina-embeddings-v3", - input=["Hello world"], - client=client, - ) - except Exception as e: - print(e) - - mock_post.assert_called_once() - - json_data = json.loads(mock_post.call_args.kwargs["data"]) - assert json_data["input"] == ["Hello world"] - assert json_data["model"] == "jina-embeddings-v3" - - -@pytest.mark.asyncio -@pytest.mark.parametrize("sync_mode", [True, False]) -async def test_lm_studio_embedding(monkeypatch, sync_mode): - monkeypatch.setenv("LM_STUDIO_API_BASE", "http://localhost:8000") - from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler - - client = HTTPHandler() if sync_mode else AsyncHTTPHandler() - with patch.object(client, "post") as mock_post: - try: - if sync_mode: - embedding( - model="lm_studio/jina-embeddings-v3", - input=["Hello world"], - client=client, - ) - else: - await litellm.aembedding( - model="lm_studio/jina-embeddings-v3", - input=["Hello world"], - client=client, - ) - except Exception as e: - print(e) - - mock_post.assert_called_once() - - json_data = json.loads(mock_post.call_args.kwargs["data"]) - assert json_data["input"] == ["Hello world"] - assert json_data["model"] == "jina-embeddings-v3" - - -@pytest.mark.parametrize( - "model", - [ - "text-embedding-ada-002", - "azure/text-embedding-ada-002", - ], -) -def test_embedding_response_ratelimit_headers(model): - response = embedding( - model=model, - input=["Hello world"], - ) - hidden_params = response._hidden_params - additional_headers = hidden_params.get("additional_headers", {}) - - print("additional_headers", additional_headers) - - # Azure is flaky with returning x-ratelimit-remaining-requests, we need to verify the upstream api returns this header - # if upstream api returns this header, we need to verify the header is transformed by litellm - if ( - "llm_provider-x-ratelimit-limit-requests" in additional_headers - or "x-ratelimit-limit-requests" in additional_headers - ): - assert "x-ratelimit-remaining-requests" in additional_headers - assert int(additional_headers["x-ratelimit-remaining-requests"]) > 0 - - assert "x-ratelimit-remaining-tokens" in additional_headers - assert int(additional_headers["x-ratelimit-remaining-tokens"]) > 0 - - -@pytest.mark.parametrize( - "input, input_type", - [ - ( - [ - "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD//gAfQ29tcHJlc3NlZCBieSBqcGVnLXJlY29tcHJlc3P/2wCEAAQEBAQEBAQEBAQGBgUGBggHBwcHCAwJCQkJCQwTDA4MDA4MExEUEA8QFBEeFxUVFx4iHRsdIiolJSo0MjRERFwBBAQEBAQEBAQEBAYGBQYGCAcHBwcIDAkJCQkJDBMMDgwMDgwTERQQDxAUER4XFRUXHiIdGx0iKiUlKjQyNEREXP/CABEIAZABkAMBIgACEQEDEQH/xAAdAAEAAQQDAQAAAAAAAAAAAAAABwEFBggCAwQJ/9oACAEBAAAAAN/gAAAAAAAAAAAAAAAAAAAAAAAAAAHTg9j6agAAp23/ADjsAAAPFrlAUYeagAAArdZ12uzcAAKax6jWUAAAAO/bna+oAC1aBxAAAAAAbM7rVABYvnRgYAAAAAbwbIABw+cMYAAAAAAvH1CuwA091RAAAAAAbpbPAGJfMXzAAAAAAJk+hdQGlmsQAAAAABk31JqBx+V1iAAAAAALp9W6gRp826AAAAAAGS/UqoGuGjwAAAAAAl76I1A1K1EAAAAAAG5G1ADUHU0AAAAAAu/1Cu4DVbTgAAAAAA3n2JAIG0IAAAAAArt3toAMV+XfEAAAAAL1uzPlQBT5qR2AAAAAenZDbm/AAa06SgAAAAerYra/LQADp+YmIAAAAC77J7Q5KAACIPnjwAAAAzbZzY24gAAGq+m4AAA7Zo2cmaoAAANWdOOAAAMl2N2TysAAAApEOj2HgAOyYtl5w5jw4zZPJyuGQ5H2AAAdes+suDUAVyfYbZTLajG8HxjgD153n3IAABH8QxxiVo4XPKpGlyTKjowvCbUAF4mD3AAACgqCzYPiPQAA900XAACmN4favRk+a9wB0xdiNAAAvU1cgAxeDcUoPdL0s1B44atQAACSs8AEewD0gM72I5jjDFiAAAPfO1QGL6z9IAlGdRgkaAAABMmRANZsSADls7k6kFW8AAAJIz4DHtW6AAk+d1jhUAAAGdyWBFcGgAX/AGnYZFgAAAM4k4CF4hAA9u3FcKi4AAAEiSEBCsRgAe3biuGxWAAACXsoAiKFgALttgs0J0AAAHpnvkBhOt4AGebE1pBtsAAAGeySA4an2wAGwEjGFxaAAAe+c+wAjKBgAyfZ3kUh3HAAAO6Yb+AKQLGgBctmb2HXDNjAAD1yzkQAENRF1gyvYG9AcI2wjgAByyuSveAAWWMcQtnoyOQs8qAPFhVh8HADt999y65gAAKKgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAf/8QAGgEBAAMBAQEAAAAAAAAAAAAAAAEFBgIEA//aAAgBAhAAAAAAAAAAAAABEAAJkBEAAB0CIAABMhyAAA6EQAAA6EQAABMiIAAAmREAAAmQiAABMgOQAEyAHIATIACIBMu7H3fT419eACEnps7DoPFQch889Wd3V2TeWIBV0o+eF8I0OrXVoAIyvBm8uDe2Wp6ADO+Mw9WDV6rSgAzvjMNWA1Op1AARlvmZbOA3NnpfSAK6iHnwfnFttZ9Wh7AeXPcB5cxWd3Wk7Pvb+uR8q+rgAAAAAAAAP//EABsBAQABBQEAAAAAAAAAAAAAAAAEAQIDBQYH/9oACAEDEAAAAAAAAAC20AL6gCNDxAArnn3gpro4AAv2l4QIgAAJWwGLVAAAX7cQYYAAFdyNZgAAAy7UazAAABsZI18UAAE6YEfWgACRNygavCACsmZkALNZjAMkqVcAC2FFoKyJWe+fMyYoMAAUw2L8t0jYzqhE0dAzd70eHj+PK7mcAa7UDN7VvBwXmDb7EAU5uw9C9KCnh2n6WoAaKIey9ODy/jN+ADRRD2fpQeY8P0QAU5zGel+gg8V53oc4AgaYTfcJ45Tx5I31wCPobQ2PpPRYuP8APMZm2kqoxQddQAAAAAAAAP/EAFMQAAEDAgIDCQkMBwUIAwAAAAECAwQFEQAGBzFREhMhMEBBYXGBCBQYIjJCRlDSFSBSVGJygpGTobHREDRDc6LBwiMzU3CyFiQlNVVkdISSlLP/2gAIAQEAAT8A/wAo74nVaBAb32bNYitfDfcS2PrURiZpU0dwVFMjN1OVY8O8u7//APkFYc076LmfSVSvmQpB/ox4QGjH/r7v/wBGR7OPCA0YH0ge7IMj2ceEBowPpA92QZHs48IDRgfSB7sgyPZx4QGjA+kD3ZBkezjwgNGB9IHuyDI9nHhAaMD6QPdkGR7OPCA0YH0ge7IMj2ceEBowPpA92QZHs48IDRgfSB7sgyPZx4QGjA+kD3ZBkezjwgNGB9IHuyDI9nHhAaMD6QPdkGR7OPCA0YH0ge7IMj2ceEBowPpA92QZHs48IDRgfSB7sgyPZx4QGjA+kD3ZBkezjwgNGB9IHuyDI9nHhAaMD6QPdkGR7OPCA0Y89fd7IMj2cN6e9GDpCTmRaOuFI9nEDSlo9qakpj5upoJNgH3d4+50JxGlxpbSH4r7bzSvJW0sLSeop5NWsw0fL8RU2rVGPDjJ4C6+4EAnYnaegYzV3StDhFcfK1LdqDuoSZBLDHWlPlqxXtNmkOulaVVxcFg3/sYA73A+kLrxKnTJrpfmSXX3jrcdWVqPWVYudvJ7nbil16s0R7vikVSVDduCVR3lNk9e5IvjKfdG5rpKmo+Yo7NXi8ALlgxJH0kiysZL0l5Uzsz/AMFn2l7m7kJ8BuSj6PnAbU8ieeZitOPPuoQ22krWtZCUpSkXJJOoDGkHui4MBT1MyW2ibITdJnuA97o/dJ1uHFczFXMyzV1Gu1N+bJV57yr7kbEjUkdA5dGlSYb7UqJIcZfaUFtuNLKFoUNRSocIONF3dBb6tih58eSCQEM1PUOqT7eELS4lK0KCkkAgg3BB4/M2Z6NlKlSKtWJiI8VoWueFS1nUhA85ZxpJ0v13Pj7kNorg0NC7tw0K4XNi3yPKPRqHqLQnpkeoD8XKmZZJVSHCG4klw/qijqQs/wCF/pwDfjc1ZqpOUKNLrVXf3qMyLJSLFbrh8ltA51qxn7P9az9V1z6istxWypMSIhRLbCD+Kj5yvUYJHCMdz7pLXWoByfWJBXUILV4bizwvRk+Z0qa4yoTodKgyZ859DEWO0t11xZslCEC5UrGlHSNOz/XVvBa26RFKkQY+xHO4v5a/UtArU3LlZptbpzm4lQ30ut7DbWk9ChwHGXq5EzHQ6ZWoCv8AdpsdDyRrIKtaFdKTwHi+6I0hrffGRKU/ZloodqSkngW5rQz1I1n1P3M2ZzJpFYyvIXdUJ0SowP8AhP8AAtI6AvitIWbWclZVqlbWElxpvcRmz+0kOcDaf5nEyXJnypM2Y8p2Q+6t11xRupa1m6lHpJ9T6B6uaVpHo7alEMz0PQnepxN0/wASRgauJ7pTNZmVynZTjuXZpzYkSRtkPDgB6UI9UZMlrgZsy1MQqxZqkRy/QHRfA4iZIaiRX5D6ghpptTi1bEIFycZmrL2YcwVitvk7ubLdfsfNClcCewcHqiiX91qbbX3yz/rGBxGmKse4ujnMz6F2dfjiGj/2VBs/ccE3J9UZOirm5ry3EQm5eqkRu3Qp0YHEd01PLGUqPT0mxk1QLV0oZaPteqdBtKNV0kUIkXah77Md6mkcH8RGBq4jupH7JyXG/wDPcP1tj1T3MuWVMQK5mt9FjJWmDGO1tHjuHqJ4nupEnvrJa+beZ4/jR6ooNGnZhrFOotNa3yXMeS02OvWo9CRwk4ytQIeWKDS6HC/V4TCWgq1itWtSz0rPCeJ7qKNenZSl2/upEtonpcShXqcC+NA+jFeW4H+1NbYKatOaswysWMaOrbscc4rujaYZuj/vzccMCpR3yehwFn+r1MAVGwGNDOhVbK4ubc4xLLFnYMB1PCNjrw/BHF58opzDk7MlHSndOSID28ja6gbtH3jChZRHqShZerOZag1S6JT3pcpzUhsahtUTwJTtJxow0G0vKRYreYS1PrIAUhNrx4yvkA+WsfCONXFnGlTLZytnqvU5KLRlvmTG2Fl/xwB0J1eookOXPkNRYUZ1991W5baaQVrWdiUi5JxkbudKzVCzOzg+abE196NWXKWOnWlvGW8p0DKMEU6g01qKzwFe5F1uEDynFnhUeO7pTJ5n0aBmyK3d+mneJVtZjOnxVfQX6ghwZtRktQ4EV6RJcNkNMoK1qOwJTcnGTe5yr9V3qXmuSKXFNj3uizkpY/0oxlbIOVslRt6oVKaZdIst9XjyHPnOK4ezkFVgw6vAmU2ewHYsllbDiFaloWNyoYz1lKZknMtRoEu6gyvdMO8zrC/IXy2j0Cs5glpg0WmyJkk+YwgrIG1WwdJxk7uap75amZyqQit6zChkLe6lueSnGWcl5ayjGEegUliKCAFuAbp5z57irqPI9NOjVOdqB31T2x7tU5KlxNryNa2CenWnDra2XFtOoUhaFFKkqFiCOAgg8qyro7zdnJwCh0Z5xi9lSVje46etarA22DGUe5spEPe5ebqgue78Ui3aj9Sl+WvFIodHoMREGj02PDjJ1NMNhAJ2m2s8m07aIHJi5WdMsxSZFiuoxG08LoGt9sDz/hjGrkzLD0hxDLDSluLISlKQSpRPMAMZU0C54zFvcidHTR4Sv2k24dI+SyPG+u2MqaBskZc3qRLimrzEftZoBaB+S0PFw0y2y2hppCUIQAEpSAAAOYAauU6XtBJmuycy5LjASVXcl05sWDu1bGxe1GHWnGXFtOoUhxCilSVAghSTYgg6iOR5eyfmXNT/AHvQKNJmKBspTaLNo+es2SntOMq9zNIc3uTm+sBoazEgWWvtdWLDGWchZTyk2E0KiR4zlrKkEbt9XW4u6uW6SNDNAzwHZ7BTTq3YkSm0XS7sS+ka/na8ZuyJmbJMwxK9T1NJJs1IR47D3S2vj2mXXlobabUtaiAlKRcknUAMZV0F56zJvT8iEKVCVY77PuhZHyWvLxlTuesl0Te3qqlysy08JMnxI4PQ0n+onEWDFhMNxokdphhsWQ20gIQkbEpFgPeyqnBg/rMhCCBfc3ur6hw4lZ1hNbpMdlbpGokhKT+OHs7zVf3EdpHzgVfzGDnGqnnbHUkYGcqqOZo/OT+VsMZ5eBG/w0K2lJKPaxDzfTJBCXFLZUTbxk3+q2GJTEhAcYdQtB1KSoEckqdLp1ThvQqnEZkxXU7lbLyAtCusKxnPubKVNU9NyhOMB03Pekm7kfsXwqRjM+jfOWUVLNZochEcapLY31gj56LgduLHZxNjjL+TM0ZpcDdCokuWL2LiEWaSflOKskYyt3M8t0tSM31hLCNZiwbLc7XVCwxljR9lHKDaRQ6Kww6BZUlQ32Qr6a7nAAHvFLSkEqUAAMT81UyGClDm/r2N6u1WKhm2oywpDKt4bPMjX/8ALC3HHCVLWSSbm+338adLhuB2O+tChzg4pOdOFDVRRbm31A/EflhiQ1IbS6y4laFaik3HJCkKBBAII4RjMOibIOYCtc/LkZD6tb0W8Zy+0luwVisdzDRX925RMyS4uxMtlD46gUFGKj3NWdY11wajSpbf71bS/qUnErQTpPjXIy2Xk7WZLCv68L0R6R2/KylO+ikK/A4Tom0jL1ZRqHa3bEXQjpPlkBGVXkDa48yj8V4p/c358lEGW/TIaOcOSCtfYG0qxSO5gp6AldczQ+9tbhsBr+NwqxRNDWjygFDjGXmpL4N99nEyVH6K/FGGmGY7SGm20oQgAJSkAJAHMAPeyJ8WEjfJD6EX1XP4DWTioZ1ZRdEBndnmWvgT2DE6tVCoE98SFFPMgGyR2DBN+E8XSq3MpToUyu7ZIK0HUcUmsRapGK46wlfBuknWnk5AOsY3I2YsNmLAagPf1HMFNp+6S68FOD9mjhV+QxUM5THrohJDKNutWHpL8halvOqWo6yokk8fT58inSESI6ylST2EbDtGKRU49VitvtkJI8tOsg7OOJA1nFSzhQKaVIkT21OA23DV3Fdu51Yk6VICCREpzznS4pKPw3WDpXk34KOgD9+fZwxpWB4JNIIG1D1/xTinaSMvylJDy3YyjwDfUXH1pviFPhTGw/FkNuoOpbagofdxU2fHhMqekOBDadus4q+bJcwqahkssfxnrOFKKjckk8iodWcpUxDySS2rgcTfWMMPtvstvNKCkLSFJI5weMzFm6mZfQUvL32UQCiOg+N1q2DFbzlWa2paXHyzGOplolKbfKOtWLnb72FUp9NeD8GU4y4OdBtfr2jGW9JTbqm4tdQlCr2D6fIPzxzYadbdQhxpYUlQBBBuCD7+pVKPTIq5D6uAcCUjWpWwYqtWlVV9Tr6yE6kIHkpHJcl1cqS5TXjfc+O3f7xxedc6IoqTAgEKnqHCdYZB5ztVsGH5D0p5x+Q6px1ZKlKUbknico5zk0J5EWWtTtPWeFOstdKejaMR5TMxhuQw4lbTiQpKkm4UD7151thtbriwlCElSidQAxXaw7VZalXsyglLadg/M8mpstcKbHko1oWDbb0duGXEOtIcQbpUkKB2g8Tm3MSMv0xbySDJduhhB+FtPQMSJD0p5yRIcK3XFFSlK1kni9HealU+UijzFjvZ5X9iVHyHDzdSve5yqqm2kU5pViuynCNnMOUZVld80lgKsVNEtns4QPqPEKNgTjOdbVWq0+tC7xmCWmRzWTrV2njEqUhQUkkEG4Ixk6ue7dFjPuuXeau08Plp5+0cP6VrS22pSiAACSdgGKpMXPnSJK/PWSBsHMOzlGRX/EmsW8koWOs3B4jONTNNoNQkIUUr3ve27awpzxb4PCTxujGpKYqkinKV4klvdJ+e3+nMkjvakS1DWtIb7FcB+7BNyTyjI67S5CDzsqP1EcRpUkqRTqfFBtvr6l9iE2/nx2V5XeeYKS9/3CEdizuD+OEm4/RnVak0+OhJtd256gm38+U5JTeY+rYyofeniNKyjv8AR0c24f8AxTx1NJTUYKhrD7Z/iGEeSP0Z63Pe8Xc6hur9dxynI7JtNeOqyAO0m/EaVv1mj/Mf/FPHU7/mEL98j8cI8gfozq2pdOZWnmdseopJ5TlKIWKShZFi8tSz2eL/AC4jSsx/Y0qR8FbqD9IA8dQmFSK1S2UjypTQ7N0L4SLJ/RmOOJVIloSk+Ijdjb4nCcEWJB5PDjrlSWWGxdS1hI7TiHHRGjsso8htCUDqSLcRpDppl5ckLABXHUl8DYBwH7jx2juAZeYmXyk7iM2t07L23I/HA/QtIWkpULggjFXgqp8+RHINkrO5O0axyfJlLK3l1F1Pit3S3cecRr7BxMqM3IjusOpCkOoKVjakixGKzTXaTU5cB4HdNOEAnzk6we0cbo3o5g0hU91FnZhCh+7T5PvM6UjfWkTmE3W0LObSnmPZyanQHqjKajMjhUeE2uANpxAhNQYzTDabNtpsOk85PXxWkjLJmRk1mGjdPR0WdA85rb9HjMqUByv1Rtgg97N2W+vYjZ1qww02y2htCQlCEhKUjUAPeLQlxCkLAUlQsQdRBxmKiOUqWopSox1m6FHht0HkjDDsl1DLKCpajYAYoFFRSYw3dlSF8K1bPkji1JCgUkXBxnjJTlJecqVOZvCWbrQn9kT/AEniqVSplYmNQoTRW4s9iRzqUeYDGXaBFoFPbiMC6/KdctYrVt/Ie+qECNMjKjyE7oLHaOkYrVEkUl8hQKmVE7hY1HkUOFInPoYjtla1bMUDLzNKb3xyy5KvKXzDoTxrjaHEKQ4gKSoWIIuCDzYzTo5WlTk2ggEG6lxr6vmH+WHmXWHFtPNqQ4k2UlQIIOwg+/y/lCq19xKm2yzFv4z7g8X6I844oOXoFBiiPDb4TYuOny1kbTxEmOxKaVHebS4hXlA4rWTpEdSnqfdxu5JR5w6tuFtONKKXEFJBsQeOShSzZIvilZTnTShySCwyfhDxj1DFPpcSmtBuM0B8JR4VK6zyCr5apFaQROiJWsCwdT4qx1KGKloseG7XSp4UnmQ+LfxJxJyLmaMoj3OU4n4TakqwrLVfSbGjy/sV4ZyhmN/yKRI+kncf6rYhaM64+QZa2YyOk7tQ7E4o+jyiU0h2SgzHhzu+R2I/PCEIbASgAJAsAOLqFFp84HvphKlkCyhwK4OnZiXkcElUKV9Fz2hh/KdZataPuwfOSoEYXQqog2MJ49Taj/LHuNVPiEj7Jf5Y9xqp8QkfZL/LHuNVPiEj7Jf5Y9xqp8QkfZL/ACx7jVT4hI+yX+WPcaqfEJH2S/yx7jVT4hI+yX+WEUCquaoTw+chQ/EYYyjWHQSpgN9K1C33XOIuR0+VMlfRbH8ziFRKdTwksRkhY89XjK+/VyWwxYf5ef/EADgRAAIBAgMDCQUHBQAAAAAAAAECAwQRAAUgMUFhEhMhIjBAUXGREDJQU6EGFDNCYoGSUnKiwdH/2gAIAQIBAT8A+L37e/wE9zHfj3k90Gk90Gk9ztqPcbd3t3e3b2129qRySGyIScRZY56ZXtwGFoKZfyX8zj7rT/JX0w+X0zbFKngcTZdLHdozyx9cbOg9pbFtENJPNYqlh4nEOWxJYykufQYVFQWRQBw1VVGk4LKAJPHxwysjFWFiNUsscKGSVwqjecVOfgErSxX/AFNhs5r2P4oHkoxHndchHKZXHFf+YpM7gnISYc0/+J0KpYhVFycUtCkQDygM/huHZZjThl59R1l97iNMsqQxvLIbKoucV1dLWykkkRg9VdOUZmyOtLO10PQhO4+Hty6mCrz7jpPu+XZsoZSp2EEYkQxyOh/KSNGf1JAipVO3rNq2EHGW1P3mkikJ6w6reYxGpd0QbyBhVCqFGwC3aV4tUycbHRnLFq+UeAUfTX9nmJhqE3BwfUYoxeqi8+1ryDVPwA0ZwCMwm4hT9Nf2eB5qobcWUfTFM3Inib9Q7QkAEnYMSvzkrv4knRn8BEkVQB0Ecg+Y15RTmCij5Qsz9c/v7KWYTQo28dDefZ5hUBI+aU9Z9vAaamnSqheF9jD0OKmmlpZWilFiNh3Eacqy9quUSSLaFDc8T4YAt7KWpNPJfap94YR1kUOhuD2NTVJTr4vuGHdpHZ3NydVVSQVaciZfIjaMVOR1URJhtKvocNSVSmzU8gP9pxHQVkhASnf9xbFJkJuHq2Fv6F/2cIiRoqIoVQLADRBUSwG6Ho3g7DiLMYX6Huh9RgTwtslT1GOdi+YnqMc7F8xP5DHOxfMT+Qxz0XzE9Rh6ymTbKD5dOJsyY3WFbcThmZiWYkk7z8W//8QAOREAAgECAgYHBwMDBQAAAAAAAQIDAAQFERITICExkQYwQVFSYXEQFCJAQlOBMlChI4KSYnJzsbL/2gAIAQMBAT8A/YCyjiwFa2PxjnWtj8Y51rY/GOda2PxjnWtj8Y51rY/GOda2PxjnWtj8Y51rY/GOda2PxjnWtj8YoMp4EHq5LlV3LvNPNI/FuXW5kcDUdw6cd4pJFkGanbJABJqacvmq7l+RR2Rgy0jiRQw2rmXM6CncOPydq+T6B4HZmfQjJ7eA+UQ6LqfMbN229V/Pyg4j1GzcnOVvlIV0pFH52bgZSt8pbRaC6TcTs3YycHvHyQBJAFQ2+WTyfgbVymlHmOI+Rjt3fe3wio4kj4Df39RNGY38jw60AscgMzSWrHe5yFJEkfBd/f1UiLIpU1JG0ZyPVJE7/pWktRxc/gUqKgyVQOtZVcZMMxUlqw3pvHdRBU5EEbIBO4CktpG3t8IpLeNOzM+fsSN5DkikmosPY75Wy8hS2duv0Z+te7wfaXlT2Nu3BSvoalsJE3xnTH81vG49UVVtzAGjbRH6cq90TxGvdE8RoW0Q7M6Cqu5VA9kVrNLvC5DvNRWEa75CWPIUqqgyVQB5bVzarMCy7n7++mUoxVhkRtW9tPdypBbRNJI3BVFYf0FdlWTErnQP24uP5JqLojgUYyNqznvZ2q46GYLKDq0khPejk/8ArOsU6HX1irTWre8xDeQBk4/FHduPtALEKozJq3skjAaQaT/wOqv4NJdco3jj6bNtby3c8VtAulJIwVRWCYJb4PbKqqGnYDWSdpPcPLZ6V9HEmikxOxjAlQaUqL9Q7x5+2xgCrrmG8/p9OrIDAg8CKkTQd07iRsdBcPV3ucSkX9H9KP1O8naIBBBG410gsBh2K3MCDKNjrE/2tSLpuqDtIFKAqhRwA6y9GVw/mAdjohEEwK2I4u0jH/Lb6exgXljL2tEwP9pq0GdzF69bfHO4fyAGx0ScPgVpl9JkB/yO309cG6w9O0ROeZq3bQnib/UOsJyBJqV9ZI7952Ogl8DDdYezfEra1B5HcdvpTfC+xicoc44QIl/t4/z7LaUTRK3bwPr1d9PoJqlPxN/A2cOvpsNvIbyA/Eh3jvHaDWHYjbYnapdWzgg/qHap7js9JseTDLZreBwbuVSAB9AP1GiSSSeJ9ltcGB8/pPEUjq6hlOYPU3FykC97dgp3aRi7HMnaw3FbzCptdaSZeJDvVh5isO6aYdcqq3gNvJ25705ikxXDJAGS/gI/5FqfHMIt10pb+H0DBjyGdYr03XRaLCojnw1sg/6FTTSzyPNNIXkc5szHMnYhuJIDmh3doPCo7+F9z5oaE0R4SrzrWR/cXnWsj+4vOtZH9xeYrWx/cXmKe6gTjID6b6lxAnMQrl5mmYsSzEkn92//2Q==" - ], - "image", - ), - (["hello world"], "text"), - ], -) -def test_cohere_img_embeddings(input, input_type): - litellm.set_verbose = True - try: - response = embedding( - model="cohere/embed-english-v3.0", - input=input, - ) - - if input_type == "image": - assert response.usage.prompt_tokens_details.image_tokens > 0 - else: - assert response.usage.prompt_tokens_details.text_tokens > 0 - except litellm.InternalServerError as e: - # Cohere API is experiencing internal server errors - this is expected - # and our exception mapping is working correctly - if "internal server error" in str(e).lower(): - pytest.skip("Cohere API is currently experiencing internal server errors") - else: - raise e - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.asyncio -async def test_embedding_with_extra_headers(sync_mode): - - input = ["hello world"] - from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler - - if sync_mode: - client = HTTPHandler() - else: - client = AsyncHTTPHandler() - - data = { - "model": "cohere/embed-english-v3.0", - "input": input, - "extra_headers": {"my-test-param": "hello-world"}, - "client": client, - } - with patch.object(client, "post") as mock_post: - try: - if sync_mode: - embedding(**data) - else: - await litellm.aembedding(**data) - except Exception as e: - print(e) - - mock_post.assert_called_once() - assert "my-test-param" in mock_post.call_args.kwargs["headers"] - - -@pytest.mark.parametrize( - "input_data, expected_payload_input", - [ - # Case 1: Input with only text strings - ( - ["hello world", "foo bar"], - ["hello world", "foo bar"], - ), - # Case 2: Input with a mix of text and a base64 encoded image - ( - [ - "A picture of a cat", - "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=", - ], - [ - {"text": "A picture of a cat"}, - { - "image": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" - }, - ], - ), - # Case 3: Input with only a base64 encoded image - ( - [ - "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" - ], - [ - { - "image": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" - } - ], - ), - ], -) -def test_jina_ai_img_embeddings(input_data, expected_payload_input): - """ - Tests the input transformation logic for Jina AI embeddings using mocks. - - This test verifies that when litellm.embedding is called with a jina_ai model, - the 'input' field in the request payload is formatted correctly based on whether - the input contains text or base64 encoded images. - """ - # We patch the `post` method of the HTTPHandler. This intercepts the network - # request before it's actually sent. - with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post") as mock_post: - # Configure the mock to return a successful, minimal valid response. - # This prevents litellm from raising an error when processing the response. - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "object": "list", - "data": [ - { - "object": "embedding", - "index": 0, - "embedding": [0.1] * 768, # Dummy embedding vector - } - ], - "model": "jina-embeddings-v4", - } - mock_post.return_value = mock_response - - # Call the function we want to test - try: - litellm.embedding(model="jina_ai/jina-embeddings-v4", input=input_data) - except Exception as e: - pytest.fail( - f"litellm.embedding call failed with an unexpected exception: {e}" - ) - - # --- Assertions --- - # 1. Check that our mock `post` method was called exactly once. - mock_post.assert_called_once() - - # 2. Extract the keyword arguments passed to the mock call. - # The request payload is in the 'data' keyword argument. - kwargs = mock_post.call_args.kwargs - assert "data" in kwargs - - # 3. Parse the JSON payload string into a Python dictionary. - sent_data = json.loads(kwargs["data"]) - - # 4. This is the core of our test: - # Assert that the 'input' field in the payload matches our expectation. - assert "input" in sent_data - assert sent_data["input"] == expected_payload_input - - -def test_encoding_format_omitted_by_default_for_openai_sdk(monkeypatch): - """ - When encoding_format is not provided, LiteLLM leaves it out of the upstream request. - - Optional global override: `LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT`. - """ - monkeypatch.delenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", raising=False) - transport = CapturingTransport( - CreateEmbeddingResponse( - object="list", - data=(Embedding(object="embedding", index=0, embedding=(0.1, 0.2, 0.3)),), - model="text-embedding-ada-002", - usage=EmbeddingUsage(prompt_tokens=1, total_tokens=1), - ) - ) - client = openai.OpenAI(api_key="sk-test", http_client=httpx.Client(transport=transport)) - - response = embedding( - model="text-embedding-ada-002", - input="Hello world", - api_key="sk-test", - client=client, - ) - - assert response.data[0]["embedding"] == [0.1, 0.2, 0.3] - assert "encoding_format" not in transport.request_bodies[0], ( - "encoding_format should be omitted from the upstream request when not provided by user" - ) - - -def test_encoding_format_explicit_value_preserved(): - """ - Test that explicitly provided encoding_format values are preserved. - - When user provides encoding_format='float' or 'base64', it should be - sent as-is to the OpenAI SDK. - """ - with patch("litellm.llms.openai.openai.OpenAIChatCompletion._get_openai_client") as mock_get_client: - # Create a mock client instance - mock_client_instance = MagicMock() - mock_get_client.return_value = mock_client_instance - - # Mock the embeddings.with_raw_response.create method - mock_response = MagicMock() - mock_response.parse.return_value = MagicMock( - model_dump=lambda: { - "data": [{"embedding": [0.1, 0.2, 0.3], "index": 0}], - "model": "text-embedding-ada-002", - "object": "list", - "usage": {"prompt_tokens": 1, "total_tokens": 1}, - } - ) - mock_response.headers = {} - - mock_client_instance.embeddings.with_raw_response.create.return_value = ( - mock_response - ) - - # Test with explicit encoding_format='float' - response = embedding( - model="text-embedding-ada-002", input="Hello world", encoding_format="float" - ) - - # Verify the encoding_format was passed correctly - call_args = mock_client_instance.embeddings.with_raw_response.create.call_args - call_kwargs = call_args[1] - - assert ( - "encoding_format" in call_kwargs - ), "encoding_format should be in the request" - assert ( - call_kwargs["encoding_format"] == "float" - ), "encoding_format should be 'float' when explicitly provided" - - print("✅ PASS: encoding_format='float' is correctly preserved") diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py deleted file mode 100644 index d42dfded27e..00000000000 --- a/tests/local_testing/test_exceptions.py +++ /dev/null @@ -1,770 +0,0 @@ -import asyncio -import os -import traceback -from typing import Any -from unittest.mock import MagicMock, patch - -import httpx -import pytest -from openai import ( - AsyncAzureOpenAI, - AsyncOpenAI, - AuthenticationError, - AzureOpenAI, - BadRequestError, - OpenAIError, -) - -import litellm -from litellm import ( # AuthenticationError,; RateLimitError,; ServiceUnavailableError,; OpenAIError, - completion, -) - -litellm.vertex_project = "litellm-ci-cd" -litellm.vertex_location = "us-central1" -litellm.num_retries = 0 - -# litellm.failure_callback = ["sentry"] -#### What this tests #### -# This tests exception mapping -> trigger an exception from an llm provider -> assert if output is of the expected type - - -# 5 providers -> OpenAI, Azure, Anthropic, Cohere, Replicate - -# 3 main types of exceptions -> - Rate Limit Errors, Context Window Errors, Auth errors (incorrect/rotated key, etc.) - -# Approach: Run each model through the test -> assert if the correct error (always the same one) is triggered - -exception_models = [ - "sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4", - "bedrock/anthropic.claude-instant-v1", -] - - -# Test 1: Context Window Errors - - -models = ["command-nightly"] - - -# for model in litellm.models_by_provider["bedrock"]: -# test_context_window(model=model) -# test_context_window(model="chat-bison") -# test_context_window_with_fallbacks(model="command-nightly") -# Test 2: InvalidAuth Errors -@pytest.mark.parametrize("model", models) -def invalid_auth(model): # set the model key to an invalid key, depending on the model - messages = [{"content": "Hello, how are you?", "role": "user"}] - temporary_key = None - try: - if model == "gpt-3.5-turbo" or model == "gpt-3.5-turbo-instruct": - temporary_key = os.environ["OPENAI_API_KEY"] - os.environ["OPENAI_API_KEY"] = "bad-key" - elif "bedrock" in model: - temporary_aws_access_key = os.environ["AWS_ACCESS_KEY_ID"] - os.environ["AWS_ACCESS_KEY_ID"] = "bad-key" - temporary_aws_region_name = os.environ["AWS_REGION_NAME"] - os.environ["AWS_REGION_NAME"] = "bad-key" - temporary_secret_key = os.environ["AWS_SECRET_ACCESS_KEY"] - os.environ["AWS_SECRET_ACCESS_KEY"] = "bad-key" - elif model == "azure/gpt-4.1-mini": - temporary_key = os.environ["AZURE_AI_API_KEY"] - os.environ["AZURE_AI_API_KEY"] = "bad-key" - elif model == "claude-3-5-haiku-20241022": - temporary_key = os.environ["ANTHROPIC_API_KEY"] - os.environ["ANTHROPIC_API_KEY"] = "bad-key" - elif model == "command-nightly": - temporary_key = os.environ["COHERE_API_KEY"] - os.environ["COHERE_API_KEY"] = "bad-key" - elif "j2" in model: - temporary_key = os.environ["AI21_API_KEY"] - os.environ["AI21_API_KEY"] = "bad-key" - elif "togethercomputer" in model: - temporary_key = os.environ["TOGETHERAI_API_KEY"] - os.environ["TOGETHERAI_API_KEY"] = "sk-test-togetherai-key-808" - elif model in litellm.openrouter_models: - temporary_key = os.environ["OPENROUTER_API_KEY"] - os.environ["OPENROUTER_API_KEY"] = "bad-key" - elif model in litellm.aleph_alpha_models: - temporary_key = os.environ["ALEPH_ALPHA_API_KEY"] - os.environ["ALEPH_ALPHA_API_KEY"] = "bad-key" - elif model in litellm.nlp_cloud_models: - os.environ["NLP_CLOUD_API_KEY"] = "bad-key" - elif ( - model - == "replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1" - ): - temporary_key = os.environ["REPLICATE_API_KEY"] - os.environ["REPLICATE_API_KEY"] = "bad-key" - print(f"model: {model}") - response = completion(model=model, messages=messages) - print(f"response: {response}") - except AuthenticationError as e: - print(f"AuthenticationError Caught Exception - {str(e)}") - except ( - OpenAIError - ) as e: # is at least an openai error -> in case of random model errors - e.g. overloaded server - print(f"OpenAIError Caught Exception - {e}") - except Exception as e: - print(type(e)) - print(type(AuthenticationError)) - print(e.__class__.__name__) - print(f"Uncaught Exception - {e}") - pytest.fail(f"Error occurred: {e}") - if temporary_key != None: # reset the key - if model == "gpt-3.5-turbo": - os.environ["OPENAI_API_KEY"] = temporary_key - elif model == "chatgpt-test": - os.environ["AZURE_AI_API_KEY"] = temporary_key - azure = True - elif model == "claude-3-5-haiku-20241022": - os.environ["ANTHROPIC_API_KEY"] = temporary_key - elif model == "command-nightly": - os.environ["COHERE_API_KEY"] = temporary_key - elif ( - model - == "replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1" - ): - os.environ["REPLICATE_API_KEY"] = temporary_key - elif "j2" in model: - os.environ["AI21_API_KEY"] = temporary_key - elif "togethercomputer" in model: - os.environ["TOGETHERAI_API_KEY"] = temporary_key - elif model in litellm.aleph_alpha_models: - os.environ["ALEPH_ALPHA_API_KEY"] = temporary_key - elif model in litellm.nlp_cloud_models: - os.environ.pop("NLP_CLOUD_API_KEY", None) - elif "bedrock" in model: - os.environ["AWS_ACCESS_KEY_ID"] = temporary_aws_access_key - os.environ["AWS_REGION_NAME"] = temporary_aws_region_name - os.environ["AWS_SECRET_ACCESS_KEY"] = temporary_secret_key - return - - -# for model in litellm.models_by_provider["bedrock"]: -# invalid_auth(model=model) -# invalid_auth(model="command-nightly") - - -# Test 3: Invalid Request Error -@pytest.mark.parametrize("model", models) -def test_invalid_request_error(model): - messages = [{"content": "hey, how's it going?", "role": "user"}] - - with pytest.raises(BadRequestError): - completion(model=model, messages=messages, max_tokens="hello world") - - -def test_completion_azure_exception(): - try: - import openai - - print("azure gpt-3.5 test\n\n") - litellm.set_verbose = True - ## Test azure call - old_azure_key = os.environ["AZURE_AI_API_KEY"] - os.environ["AZURE_AI_API_KEY"] = "good morning" - response = completion( - model="azure/gpt-4.1-mini", - messages=[{"role": "user", "content": "hello"}], - ) - os.environ["AZURE_AI_API_KEY"] = old_azure_key - print(f"response: {response}") - print(response) - except openai.AuthenticationError as e: - os.environ["AZURE_AI_API_KEY"] = old_azure_key - print("good job got the correct error for azure when key not set") - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_completion_azure_exception() - - -async def asynctest_completion_azure_exception(): - try: - import openai - - import litellm - - print("azure gpt-3.5 test\n\n") - litellm.set_verbose = True - ## Test azure call - old_azure_key = os.environ["AZURE_AI_API_KEY"] - os.environ["AZURE_AI_API_KEY"] = "good morning" - response = await litellm.acompletion( - model="azure/gpt-4.1-mini", - messages=[{"role": "user", "content": "hello"}], - ) - print(f"response: {response}") - print(response) - except openai.AuthenticationError as e: - os.environ["AZURE_AI_API_KEY"] = old_azure_key - print("good job got the correct error for azure when key not set") - print(e) - except Exception as e: - print("Got wrong exception") - print("exception", e) - pytest.fail(f"Error occurred: {e}") - - -# import asyncio -# asyncio.run( -# asynctest_completion_azure_exception() -# ) - - -def asynctest_completion_openai_exception_bad_model(): - try: - import asyncio - - import openai - - import litellm - - print("azure exception bad model\n\n") - litellm.set_verbose = True - - ## Test azure call - async def test(): - response = await litellm.acompletion( - model="openai/gpt-6", - messages=[{"role": "user", "content": "hello"}], - ) - - asyncio.run(test()) - except openai.NotFoundError: - print("Good job this is a NotFoundError for a model that does not exist!") - print("Passed") - except Exception as e: - print("Raised wrong type of exception", type(e)) - pytest.fail(f"Error occurred: {e}") - - -# asynctest_completion_openai_exception_bad_model() - - -def asynctest_completion_azure_exception_bad_model(): - try: - import asyncio - - import openai - - import litellm - - print("azure exception bad model\n\n") - litellm.set_verbose = True - - ## Test azure call - async def test(): - response = await litellm.acompletion( - model="azure/gpt-12", - messages=[{"role": "user", "content": "hello"}], - ) - - asyncio.run(test()) - except openai.NotFoundError: - print("Good job this is a NotFoundError for a model that does not exist!") - print("Passed") - except Exception as e: - print("Raised wrong type of exception", type(e)) - pytest.fail(f"Error occurred: {e}") - - -# asynctest_completion_azure_exception_bad_model() - - -def test_completion_openai_exception(): - # test if openai:gpt raises openai.AuthenticationError - try: - import openai - - print("openai gpt-3.5 test\n\n") - litellm.set_verbose = True - ## Test azure call - old_azure_key = os.environ["OPENAI_API_KEY"] - os.environ["OPENAI_API_KEY"] = "good morning" - response = completion( - model="gpt-4", - messages=[{"role": "user", "content": "hello"}], - ) - print(f"response: {response}") - print(response) - except openai.AuthenticationError as e: - os.environ["OPENAI_API_KEY"] = old_azure_key - print("OpenAI: good job got the correct error for openai when key not set") - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_completion_openai_exception() - - -def test_completion_mistral_exception(): - # test if mistral/mistral-tiny raises openai.AuthenticationError - try: - import openai - - print("Testing mistral ai exception mapping") - litellm.set_verbose = True - ## Test azure call - old_azure_key = os.environ["MISTRAL_API_KEY"] - os.environ["MISTRAL_API_KEY"] = "good morning" - response = completion( - model="mistral/mistral-tiny", - messages=[{"role": "user", "content": "hello"}], - ) - print(f"response: {response}") - print(response) - except openai.AuthenticationError as e: - os.environ["MISTRAL_API_KEY"] = old_azure_key - print("good job got the correct error for openai when key not set") - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_completion_mistral_exception() - - -def test_content_policy_violation_error_streaming(): - """ - Production Test. - """ - litellm.set_verbose = False - print("test_async_completion with stream") - - async def test_get_response(): - try: - response = await litellm.acompletion( - model="azure/gpt-4.1-mini", - messages=[{"role": "user", "content": "say 1"}], - temperature=0, - top_p=1, - stream=True, - max_tokens=512, - presence_penalty=0, - frequency_penalty=0, - ) - print(f"response: {response}") - - num_finish_reason = 0 - async for chunk in response: - print(chunk) - if chunk["choices"][0].get("finish_reason") is not None: - num_finish_reason += 1 - print("finish_reason", chunk["choices"][0].get("finish_reason")) - - assert ( - num_finish_reason == 1 - ), f"expected only one finish reason. Got {num_finish_reason}" - except Exception as e: - pytest.fail(f"GOT exception for gpt-3.5 instruct In streaming{e}") - - asyncio.run(test_get_response()) - - async def test_get_error(): - try: - response = await litellm.acompletion( - model="azure/gpt-4.1-mini", - messages=[ - {"role": "user", "content": "where do i buy lethal drugs from"} - ], - temperature=0, - top_p=1, - stream=True, - max_tokens=512, - presence_penalty=0, - frequency_penalty=0, - mock_response="Exception: content_filter_policy", - ) - print(f"response: {response}") - - num_finish_reason = 0 - async for chunk in response: - print(chunk) - if chunk["choices"][0].get("finish_reason") is not None: - num_finish_reason += 1 - print("finish_reason", chunk["choices"][0].get("finish_reason")) - - pytest.fail("Expected a content-policy error in streaming, got a clean stream") - except Exception as e: - pass - - asyncio.run(test_get_error()) - - -# test_completion_perplexity_exception_on_openai_client() - - -def test_completion_perplexity_exception(monkeypatch): - import openai - - print("perplexity test\n\n") - litellm.set_verbose = True - monkeypatch.setenv("PERPLEXITYAI_API_KEY", "good morning") - with pytest.raises(openai.AuthenticationError, match="PerplexityException"): - completion( - model="perplexity/mistral-7b-instruct", - messages=[{"role": "user", "content": "hello"}], - ) - - -def test_completion_openai_api_key_exception(monkeypatch): - import openai - - print("gpt-3.5 test\n\n") - litellm.set_verbose = True - monkeypatch.setenv("OPENAI_API_KEY", "good morning") - with pytest.raises(openai.AuthenticationError, match="OpenAIException"): - completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "hello"}], - ) - - -# tesy_async_acompletion() - - -def test_router_completion_vertex_exception(): - try: - import litellm - - litellm.set_verbose = True - router = litellm.Router( - model_list=[ - { - "model_name": "vertex-gemini-pro", - "litellm_params": { - "model": "vertex_ai/gemini-pro", - "api_key": "good-morning", - }, - }, - ] - ) - response = router.completion( - model="vertex-gemini-pro", - messages=[{"role": "user", "content": "hello"}], - vertex_project="bad-project", - ) - pytest.fail("Request should have failed - bad api key") - except Exception as e: - print("exception: ", e) - - -def test_litellm_completion_vertex_exception(): - try: - import litellm - - litellm.set_verbose = True - response = completion( - model="vertex_ai/gemini-pro", - api_key="good-morning", - messages=[{"role": "user", "content": "hello"}], - vertex_project="bad-project", - ) - pytest.fail("Request should have failed - bad api key") - except Exception as e: - print("exception: ", e) - - -def test_litellm_predibase_exception(): - """ - Test - Assert that the Predibase API Key is not returned on Authentication Errors - """ - try: - import litellm - - litellm.set_verbose = True - response = completion( - model="predibase/llama-3-8b-instruct", - messages=[{"role": "user", "content": "What is the meaning of life?"}], - tenant_id="c4768f95", - api_key="hf-rawapikey", - ) - pytest.fail("Request should have failed - bad api key") - except Exception as e: - if "hf-rawapikey" in str(e): - pytest.fail("predibase error leaked the raw api key") - print("exception: ", e) - - -# # test_invalid_request_error(model="command-nightly") -# # Test 3: Rate Limit Errors -# def test_model_call(model): -# try: -# sample_text = "how does a court case get to the Supreme Court?" -# messages = [{ "content": sample_text,"role": "user"}] -# print(f"model: {model}") -# response = completion(model=model, messages=messages) -# except RateLimitError as e: -# print(f"headers: {e.response.headers}") -# return True -# # except OpenAIError: # is at least an openai error -> in case of random model errors - e.g. overloaded server -# # return True -# except Exception as e: -# print(f"Uncaught Exception {model}: {type(e).__name__} - {e}") -# traceback.print_exc() -# pass -# return False -# # Repeat each model 500 times -# # extended_models = [model for model in models for _ in range(250)] -# extended_models = ["azure/gpt-4.1-mini" for _ in range(250)] - -# def worker(model): -# return test_model_call(model) - -# # Create a dictionary to store the results -# counts = {True: 0, False: 0} - -# # Use Thread Pool Executor -# with ThreadPoolExecutor(max_workers=500) as executor: -# # Use map to start the operation in thread pool -# results = executor.map(worker, extended_models) - -# # Iterate over results and count True/False -# for result in results: -# counts[result] += 1 - -# accuracy_score = counts[True]/(counts[True] + counts[False]) -# print(f"accuracy_score: {accuracy_score}") - - -def test_anthropic_tool_calling_exception(): - """ - Related - https://github.com/BerriAI/litellm/issues/4348 - """ - tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": {}, - }, - } - ] - try: - litellm.completion( - model="claude-haiku-4-5-20251001", - messages=[{"role": "user", "content": "Hey, how's it going?"}], - tools=tools, - ) - except litellm.BadRequestError: - pass - - -from typing import Optional, Union - -from openai import OpenAI - - -def _pre_call_utils( - call_type: str, - data: dict, - client: Union[OpenAI, AsyncOpenAI], - sync_mode: bool, - streaming: Optional[bool], -): - if call_type == "embedding": - data["input"] = "Hello world!" - if isinstance(client, (AzureOpenAI, AsyncAzureOpenAI)): - mapped_target: Any = client.embeddings.with_raw_response - patched_attr = "create" - else: - mapped_target = client - patched_attr = "post" - if sync_mode: - original_function = litellm.embedding - else: - original_function = litellm.aembedding - elif call_type == "chat_completion": - data["messages"] = [{"role": "user", "content": "Hello world"}] - if streaming is True: - data["stream"] = True - mapped_target = client.chat.completions.with_raw_response # type: ignore - patched_attr = "create" - if sync_mode: - original_function = litellm.completion - else: - original_function = litellm.acompletion - elif call_type == "completion": - data["prompt"] = "Hello world" - if streaming is True: - data["stream"] = True - mapped_target = client.completions.with_raw_response # type: ignore - patched_attr = "create" - if sync_mode: - original_function = litellm.text_completion - else: - original_function = litellm.atext_completion - - return data, original_function, mapped_target, patched_attr - - -@pytest.mark.parametrize( - "sync_mode", - [True, False], -) -@pytest.mark.parametrize( - "provider, model, call_type, streaming", - [ - ("azure", "azure/text-embedding-ada-002", "embedding", True), - ("azure", "azure_text/gpt-3.5-turbo-instruct", "completion", True), - ], -) -@pytest.mark.asyncio -async def test_exception_with_headers(sync_mode, provider, model, call_type, streaming): - """ - User feedback: litellm says "No deployments available for selected model, Try again in 60 seconds" - but Azure says to retry in at most 9s - - ``` - {"message": "litellm.proxy.proxy_server.embeddings(): Exception occured - No deployments available for selected model, Try again in 60 seconds. Passed model=text-embedding-ada-002. pre-call-checks=False, allowed_model_region=n/a, cooldown_list=[('b49cbc9314273db7181fe69b1b19993f04efb88f2c1819947c538bac08097e4c', {'Exception Received': 'litellm.RateLimitError: AzureException RateLimitError - Requests to the Embeddings_Create Operation under Azure OpenAI API version 2023-09-01-preview have exceeded call rate limit of your current OpenAI S0 pricing tier. Please retry after 9 seconds. Please go here: https://aka.ms/oai/quotaincrease if you would like to further increase the default rate limit.', 'Status Code': '429'})]", "level": "ERROR", "timestamp": "2024-08-22T03:25:36.900476"} - ``` - """ - print(f"Received args: {locals()}") - import openai - - if sync_mode: - if provider == "openai": - openai_client = openai.OpenAI(api_key="") - elif provider == "azure": - openai_client = openai.AzureOpenAI( - api_key="", base_url="", api_version=litellm.AZURE_DEFAULT_API_VERSION - ) - else: - if provider == "openai": - openai_client = openai.AsyncOpenAI(api_key="") - elif provider == "azure": - openai_client = openai.AsyncAzureOpenAI( - api_key="", base_url="", api_version=litellm.AZURE_DEFAULT_API_VERSION - ) - - data = {"model": model} - data, original_function, mapped_target, patched_attr = _pre_call_utils( - call_type=call_type, - data=data, - client=openai_client, - sync_mode=sync_mode, - streaming=streaming, - ) - - cooldown_time = 30.0 - - def _return_exception(*args, **kwargs): - - from httpx import Headers, Request, Response - - kwargs = { - "request": Request("POST", "https://www.google.com"), - "message": "Error code: 429 - Rate Limit Error!", - "body": {"detail": "Rate Limit Error!"}, - "code": None, - "param": None, - "type": None, - "response": Response( - status_code=429, - headers=Headers( - { - "date": "Sat, 21 Sep 2024 22:56:53 GMT", - "server": "uvicorn", - "retry-after": "30", - "content-length": "30", - "content-type": "application/json", - } - ), - request=Request("POST", "http://0.0.0.0:9000/chat/completions"), - ), - "status_code": 429, - "request_id": None, - } - - exception = Exception() - for k, v in kwargs.items(): - setattr(exception, k, v) - raise exception - - with patch.object( - mapped_target, - patched_attr, - side_effect=_return_exception, - ): - new_retry_after_mock_client = MagicMock(return_value=-1) - - litellm.utils._get_retry_after_from_exception_header = ( - new_retry_after_mock_client - ) - - async def call_and_drain(): - if sync_mode: - resp = original_function(**data, client=openai_client) - if streaming: - for chunk in resp: - continue - else: - resp = await original_function(**data, client=openai_client) - - if streaming: - async for chunk in resp: - continue - - with pytest.raises(litellm.RateLimitError) as exc_info: - await call_and_drain() - - assert exc_info.value.litellm_response_headers is not None - assert int(exc_info.value.litellm_response_headers["retry-after"]) == cooldown_time - - -@pytest.mark.asyncio -@pytest.mark.parametrize("model", ["azure/gpt-4.1-mini", "openai/gpt-3.5-turbo"]) -async def test_bad_request_error_contains_httpx_response(model): - """ - Test that the BadRequestError contains the httpx response - - Relevant issue: https://github.com/BerriAI/litellm/issues/6732 - """ - with pytest.raises(litellm.BadRequestError) as exc_info: - await litellm.acompletion( - model=model, - messages=[{"role": "user", "content": "Hello world"}], - bad_arg="bad_arg", - ) - e = exc_info.value - print("e.response", e.response) - print("vars(e.response)", vars(e.response)) - assert e.response is not None - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.parametrize("stream_mode", [True, False]) -@pytest.mark.parametrize("model", ["gpt-4.1-nano"]) # "gpt-4o-mini", -@pytest.mark.asyncio -async def test_exception_bubbling_up(sync_mode, stream_mode, model): - """ - make sure code, param, and type are bubbled up - """ - import litellm - - litellm.set_verbose = True - async def _call_with_bad_role(): - if sync_mode: - litellm.completion( - model=model, - messages=[{"role": "usera", "content": "hi"}], - stream=stream_mode, - sync_stream=sync_mode, - ) - else: - await litellm.acompletion( - model=model, - messages=[{"role": "usera", "content": "hi"}], - stream=stream_mode, - sync_stream=sync_mode, - ) - - with pytest.raises(Exception, match='litellm\\.BadRequestError: OpenAIException - Invalid value') as exc_info: - await _call_with_bad_role() - - assert exc_info.value.code == "invalid_value" - assert exc_info.value.param is not None - assert exc_info.value.type == "invalid_request_error" diff --git a/tests/local_testing/test_fake_openai_endpoint.py b/tests/local_testing/test_fake_openai_endpoint.py deleted file mode 100644 index df09657acbc..00000000000 --- a/tests/local_testing/test_fake_openai_endpoint.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Regression tests for the canned OpenAI mock that decouples the suite from a -hosted endpoint (see ``tests/fake_openai_endpoint.py`` and -``tests/_fake_openai_endpoint_server.py``). - -Before this, router/completion/triton tests hardcoded a shared Railway mock as -their ``api_base``; when that single deployment went down, unrelated CI jobs -failed with ``404 Application not found``. These tests pin the two behaviors -those tests now rely on from the local stand-in (the Triton embeddings shape and -the ``slow-endpoint`` delay) and guard against the dead host creeping back in. -""" - -from __future__ import annotations - -import re -from pathlib import Path -from typing import Final - -import httpx -import pytest -from openai.types import ModerationCreateResponse - -from tests.fake_openai_endpoint import ( - _LOCAL_DEFAULT, - _resolve_base, - ensure_fake_openai_endpoint, -) - -_REPO_ROOT = Path(__file__).resolve().parents[2] - -_MIGRATED_FILES = ( - "tests/llm_translation/test_triton.py", - "tests/local_testing/test_completion.py", -) - -# A live ``railway.app/`` or ``railway.app"`` api_base reappearing in a migrated -# file would re-couple CI to an external service's uptime. The deliberately -# broken ``...railway.appzzzzz`` fallback URL is not a live host, so it does not -# match. -_LIVE_HOSTED_MOCK = re.compile(r"railway\.app(?:/|\")") - - -def test_chat_completion_shape(): - base = ensure_fake_openai_endpoint() - response = httpx.post( - f"{base}/v1/chat/completions", - json={"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]}, - timeout=10, - ) - assert response.status_code == 200 - body = response.json() - assert body["choices"][0]["message"]["content"] - assert body["usage"]["total_tokens"] == 40 - - -def test_moderations_route_parses_as_an_openai_response(): - base: Final = ensure_fake_openai_endpoint() - response: Final = httpx.post( - f"{base}/v1/moderations", - json={"input": ["I want to harm someone", "hello"], "model": "omni-moderation-latest"}, - timeout=10, - ) - assert response.status_code == 200 - parsed: Final = ModerationCreateResponse.model_validate(response.json()) - assert parsed.model == "omni-moderation-latest" - assert len(parsed.results) == 2 - assert parsed.results[0].categories.violence is False - - -def test_triton_embeddings_route(): - base = ensure_fake_openai_endpoint() - response = httpx.post(f"{base}/triton/embeddings", json={"inputs": []}, timeout=10) - assert response.status_code == 200 - output = response.json()["outputs"][0] - assert output["shape"] == [1, 2] - assert output["data"] == [0.1, 0.2] - - -def test_slow_model_blocks_past_client_timeout(): - base = ensure_fake_openai_endpoint() - with pytest.raises(httpx.TimeoutException): - httpx.post( - f"{base}/v1/chat/completions", - json={ - "model": "slow-endpoint", - "messages": [{"role": "user", "content": "hi"}], - }, - timeout=0.5, - ) - - -def test_remote_env_base_resolves_to_local(monkeypatch): - monkeypatch.setenv( - "FAKE_OPENAI_API_BASE", - "https://exampleopenaiendpoint-production.up.railway.app", - ) - assert _resolve_base() == _LOCAL_DEFAULT - - -@pytest.mark.parametrize("host", ["127.0.0.1", "localhost", "[::1]"]) -def test_loopback_env_base_is_honored(monkeypatch, host): - base = f"http://{host}:9191" - monkeypatch.setenv("FAKE_OPENAI_API_BASE", base) - assert _resolve_base() == base - - -@pytest.mark.parametrize("relative_path", _MIGRATED_FILES) -def test_migrated_files_have_no_live_hosted_mock(relative_path): - source = (_REPO_ROOT / relative_path).read_text() - assert not _LIVE_HOSTED_MOCK.search(source), ( - f"{relative_path} references the hosted Railway mock; point api_base at " - "FAKE_OPENAI_API_BASE so CI does not depend on an external service" - ) diff --git a/tests/local_testing/test_function_call_parsing.py b/tests/local_testing/test_function_call_parsing.py deleted file mode 100644 index 6c1d1c7c5af..00000000000 --- a/tests/local_testing/test_function_call_parsing.py +++ /dev/null @@ -1,144 +0,0 @@ -# What is this? -## Test to make sure function call response always works with json.loads() -> no extra parsing required. Relevant issue - https://github.com/BerriAI/litellm/issues/2654 -import traceback - -from dotenv import load_dotenv - -load_dotenv() -import io - -import json -import warnings -from typing import List - -import pytest - -import litellm -from litellm import completion - - -# Just a stub to keep the sample code simple -class Trade: - def __init__(self, order: dict): - self.order = order - - @staticmethod - def buy(order: dict): - return Trade(order) - - @staticmethod - def sell(order: dict): - return Trade(order) - - -def trade(model_name: str) -> List[Trade]: # type: ignore - def parse_order(order: dict) -> Trade: - action = order["action"] - - if action == "buy": - return Trade.buy(order) - elif action == "sell": - return Trade.sell(order) - else: - raise ValueError(f"Invalid action {action}") - - def parse_call(call) -> List[Trade]: - arguments = json.loads(call.function.arguments) - - trades = [parse_order(order) for order in arguments["orders"]] - return trades - - tool_spec = { - "type": "function", - "function": { - "name": "trade", - "description": "Execute orders to manage the portfolio. Orders will be executed immediately at the stated prices.", - "parameters": { - "type": "object", - "properties": { - "orders": { - "type": "array", - "items": { - "type": "object", - "properties": { - "action": {"type": "string", "enum": ["buy", "sell"]}, - "asset": {"type": "string"}, - "amount": { - "type": "number", - "description": "Amount of asset to buy or sell.", - }, - }, - "required": ["action", "asset", "amount"], - }, - }, - }, - }, - }, - } - - try: - response = completion( - model_name, - [ - { - "role": "system", - "content": """You are an expert asset manager, managing a portfolio. - - Always use the `trade` function. Make sure that you call it correctly. For example, the following is a valid call: - ``` - trade({ - "orders": [ - {"action": "buy", "asset": "BTC", "amount": 0.1}, - {"action": "sell", "asset": "ETH", "amount": 0.2} - ] - }) - ``` - - If there are no trades to make, call `trade` with an empty array: - ``` - trade({ "orders": [] }) - ``` - """, - }, - { - "role": "user", - "content": """Manage the portfolio. - - Don't jabber. - - This is the current market data: - ``` - {market_data} - ``` - - Your portfolio is as follows: - ``` - {portfolio} - ``` - """.replace( - "{market_data}", "BTC: 64,000 USD\nETH: 3,500 USD" - ).replace( - "{portfolio}", "USD: 1000, BTC: 0.1, ETH: 0.2" - ), - }, - ], - tools=[tool_spec], - tool_choice={ - "type": "function", - "function": {"name": tool_spec["function"]["name"]}, # type: ignore - }, - ) - calls = response.choices[0].message.tool_calls - trades = [trade for call in calls for trade in parse_call(call)] - return trades - except litellm.InternalServerError: - pass - - -@pytest.mark.parametrize( - "model", ["us.anthropic.claude-haiku-4-5-20251001-v1:0"] -) -@pytest.mark.flaky(retries=6, delay=10) -def test_function_call_parsing(model): - trades = trade(model) - print([trade.order for trade in trades if trade is not None]) diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py deleted file mode 100644 index d400b7062cf..00000000000 --- a/tests/local_testing/test_function_calling.py +++ /dev/null @@ -1,302 +0,0 @@ -import traceback - -from dotenv import load_dotenv - -load_dotenv() -import io -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -import litellm -from litellm import RateLimitError, Timeout, completion_cost, embedding - -litellm.num_retries = 0 -litellm.cache = None -# litellm.set_verbose=True -import json - -# litellm.success_callback = ["langfuse"] - - -def get_current_weather(location, unit="fahrenheit"): - """Get the current weather in a given location""" - if "tokyo" in location.lower(): - return json.dumps({"location": "Tokyo", "temperature": "10", "unit": "celsius"}) - elif "san francisco" in location.lower(): - return json.dumps( - {"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"} - ) - elif "paris" in location.lower(): - return json.dumps({"location": "Paris", "temperature": "22", "unit": "celsius"}) - else: - return json.dumps({"location": location, "temperature": "unknown"}) - - -# Example dummy function hard coded to return the same weather - - -# In production, this could be your backend API or an external API -# test_parallel_function_call() - - -from litellm.types.utils import ChatCompletionMessageToolCall, Function, Message - -_PARALLEL_TOOL_HISTORY_MESSAGES = [ - { - "role": "user", - "content": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses", - }, - Message( - content="Here are the current weather conditions for San Francisco, Tokyo, and Paris:", - role="assistant", - tool_calls=[ - ChatCompletionMessageToolCall( - index=1, - function=Function( - arguments='{"location": "San Francisco, CA", "unit": "fahrenheit"}', - name="get_current_weather", - ), - id="tooluse_Jj98qn6xQlOP_PiQr-w9iA", - type="function", - ) - ], - function_call=None, - ), - { - "tool_call_id": "tooluse_Jj98qn6xQlOP_PiQr-w9iA", - "role": "tool", - "name": "get_current_weather", - "content": '{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}', - }, -] - - -@pytest.mark.parametrize( - "model, messages", - [ - # Anthropic Messages API: a dummy tool is injected without modify_params, - # so tool history with no tools= completes instead of raising. - ("claude-haiku-4-5-20251001", _PARALLEL_TOOL_HISTORY_MESSAGES), - ( - "us.anthropic.claude-sonnet-4-5-20250929-v1:0", - [ - { - "role": "user", - "content": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses", - } - ], - ), - ( - "claude-haiku-4-5-20251001", - [ - { - "role": "user", - "content": "What's the weather like in San Francisco, Tokyo, and Paris? - give me 3 responses", - } - ], - ), - ], -) -def test_parallel_function_call_anthropic_error_msg(model, messages): - """ - Tool history without an explicit ``tools`` param must complete, not raise. - - Anthropic (and Bedrock Invoke via ``AnthropicConfig.transform_request``) - inject a dummy tool so CLIs work with ``modify_params`` left off. Bedrock - Converse's no-raise behavior is covered offline in - ``tests/unit/llms/bedrock/chat/test_converse_transformation.py`` - (see #24158, #27138), which needs no live credentials. - """ - # Force modify_params off as a clean baseline: it exercises the Anthropic - # dummy-tool path, which injects regardless of modify_params - # (other tests in this file set it to True and don't reset it) - original_modify_params = litellm.modify_params - litellm.modify_params = False - try: - litellm.set_verbose = True - second_response = litellm.completion( - model=model, - messages=messages, - temperature=0.2, - seed=22, - drop_params=True, - ) # get a new response from the model where it can see the function response - print("second response\n", second_response) - except litellm.InternalServerError as e: - print(e) - except litellm.RateLimitError as e: - print(e) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - finally: - litellm.modify_params = original_modify_params - - -def test_parallel_function_call_stream(): - try: - litellm.set_verbose = True - # Step 1: send the conversation and available functions to the model - messages = [ - { - "role": "user", - "content": "What's the weather like in San Francisco, Tokyo, and Paris?", - } - ] - tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - }, - }, - "required": ["location"], - }, - }, - } - ] - response = litellm.completion( - model="gpt-6-luna", - messages=messages, - tools=tools, - stream=True, - tool_choice="auto", # auto is default, but we'll be explicit - complete_response=True, - ) - print("Response\n", response) - # for chunk in response: - # print(chunk) - response_message = response.choices[0].message - tool_calls = response_message.tool_calls - - print("length of tool calls", len(tool_calls)) - print("Expecting there to be 3 tool calls") - assert ( - len(tool_calls) > 1 - ) # this has to call the function for SF, Tokyo and parise - - # Step 2: check if the model wanted to call a function - if tool_calls: - # Step 3: call the function - # Note: the JSON response may not always be valid; be sure to handle errors - available_functions = { - "get_current_weather": get_current_weather, - } # only one function in this example, but you can have multiple - messages.append( - response_message - ) # extend conversation with assistant's reply - print("Response message\n", response_message) - # Step 4: send the info for each function call and function response to the model - for tool_call in tool_calls: - function_name = tool_call.function.name - function_to_call = available_functions[function_name] - function_args = json.loads(tool_call.function.arguments) - function_response = function_to_call( - location=function_args.get("location"), - unit=function_args.get("unit"), - ) - messages.append( - { - "tool_call_id": tool_call.id, - "role": "tool", - "name": function_name, - "content": function_response, - } - ) # extend conversation with function response - print(f"messages: {messages}") - second_response = litellm.completion( - model="gpt-6-luna", messages=messages, temperature=0.2, seed=22, reasoning_effort="none" - ) # get a new response from the model where it can see the function response - print("second response\n", second_response) - return second_response - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_parallel_function_call_stream() - - - - -@pytest.mark.parametrize("sync_mode", [False]) -@pytest.mark.asyncio -@pytest.mark.flaky(retries=6, delay=1) -async def test_watsonx_tool_choice(sync_mode, monkeypatch): - import json - - from litellm import acompletion, completion - from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler - - # Mock the IAM token generation to avoid actual API calls - monkeypatch.setenv("WATSONX_API_KEY", "mock-api-key") - monkeypatch.setenv("WATSONX_TOKEN", "mock-watsonx-token") - monkeypatch.setenv("WATSONX_API_BASE", "https://us-south.ml.cloud.ibm.com") - monkeypatch.setenv("WATSONX_PROJECT_ID", "mock-project-id") - - monkeypatch.setattr(litellm, "set_verbose", True) - tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - }, - } - ] - messages = [{"role": "user", "content": "What is the weather in San Francisco?"}] - - client = HTTPHandler() if sync_mode else AsyncHTTPHandler() - with patch.object(client, "post", return_value=MagicMock()) as mock_completion: - try: - if sync_mode: - resp = completion( - model="watsonx/meta-llama/llama-3-1-8b-instruct", - messages=messages, - tools=tools, - tool_choice="auto", - client=client, - ) - else: - resp = await acompletion( - model="watsonx/meta-llama/llama-3-1-8b-instruct", - messages=messages, - tools=tools, - tool_choice="auto", - client=client, - stream=True, - ) - - print(resp) - - mock_completion.assert_called_once() - print(mock_completion.call_args.kwargs) - json_data = json.loads(mock_completion.call_args.kwargs["data"]) - assert json_data["tool_choice_option"] == "auto" - except Exception as e: - print(e) - if "The read operation timed out" in str(e): - pytest.skip("Skipping test due to timeout") - else: - raise e diff --git a/tests/local_testing/test_get_optional_params_functions_not_supported.py b/tests/local_testing/test_get_optional_params_functions_not_supported.py deleted file mode 100644 index c4580adba23..00000000000 --- a/tests/local_testing/test_get_optional_params_functions_not_supported.py +++ /dev/null @@ -1,29 +0,0 @@ -import litellm -from litellm import get_optional_params - -litellm.add_function_to_prompt = True -optional_params = get_optional_params( - model="", - tools=[ - { - "type": "function", - "function": { - "description": "Get the current weather in a given location", - "name": "get_current_weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - }, - } - ], - tool_choice="auto", -) -assert optional_params is not None diff --git a/tests/local_testing/test_google_ai_studio_gemini.py b/tests/local_testing/test_google_ai_studio_gemini.py deleted file mode 100644 index 43b64ded1ab..00000000000 --- a/tests/local_testing/test_google_ai_studio_gemini.py +++ /dev/null @@ -1,36 +0,0 @@ -import os, sys, traceback - -import litellm -from dotenv import load_dotenv - - -def generate_text(): - try: - litellm.set_verbose = True - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is this image?"}, - { - "type": "image_url", - "image_url": { - "url": "https://avatars.githubusercontent.com/u/17561003?v=4" - }, - }, - ], - } - ] - response = litellm.completion( - model="gemini/gemini-pro-vision", - messages=messages, - stop="Hello world", - num_retries=3, - ) - print(response) - assert isinstance(response.choices[0].message.content, str) == True - except Exception as exception: - raise Exception("An error occurred during text generation:", exception) - - -# generate_text() diff --git a/tests/local_testing/test_img_resize.py b/tests/local_testing/test_img_resize.py deleted file mode 100644 index c8218773cb6..00000000000 --- a/tests/local_testing/test_img_resize.py +++ /dev/null @@ -1,78 +0,0 @@ -from typing import Literal - - -def calculate_img_tokens( - width, - height, - mode: Literal["low", "high", "auto"] = "auto", - base_tokens: int = 85, # openai default - https://openai.com/pricing -): - if mode == "low": - return base_tokens - elif mode == "high" or mode == "auto": - resized_width, resized_height = resize_image_high_res( - width=width, height=height - ) - tiles_needed_high_res = calculate_tiles_needed(resized_width, resized_height) - tile_tokens = (base_tokens * 2) * tiles_needed_high_res - total_tokens = base_tokens + tile_tokens - return total_tokens - - -def resize_image_high_res(width, height): - # Maximum dimensions for high res mode - max_short_side = 768 - max_long_side = 2000 - - # Determine the longer and shorter sides - longer_side = max(width, height) - shorter_side = min(width, height) - - # Calculate the aspect ratio - aspect_ratio = longer_side / shorter_side - - # Resize based on the short side being 768px - if width <= height: # Portrait or square - resized_width = max_short_side - resized_height = int(resized_width * aspect_ratio) - # if the long side exceeds the limit after resizing, adjust both sides accordingly - if resized_height > max_long_side: - resized_height = max_long_side - resized_width = int(resized_height / aspect_ratio) - else: # Landscape - resized_height = max_short_side - resized_width = int(resized_height * aspect_ratio) - # if the long side exceeds the limit after resizing, adjust both sides accordingly - if resized_width > max_long_side: - resized_width = max_long_side - resized_height = int(resized_width / aspect_ratio) - - return resized_width, resized_height - - -# Test the function with the given example -def calculate_tiles_needed( - resized_width, resized_height, tile_width=512, tile_height=512 -): - tiles_across = (resized_width + tile_width - 1) // tile_width - tiles_down = (resized_height + tile_height - 1) // tile_height - total_tiles = tiles_across * tiles_down - return total_tiles - - -# Test high res mode with 1875 x 768 image -resized_width_high_res = 1875 -resized_height_high_res = 768 -tiles_needed_high_res = calculate_tiles_needed( - resized_width_high_res, resized_height_high_res -) -print( - f"Tiles needed for high res image ({resized_width_high_res}x{resized_height_high_res}): {tiles_needed_high_res}" -) - -# If you had the original size and needed to resize and then calculate tiles: -original_size = (10000, 4096) -resized_size_high_res = resize_image_high_res(*original_size) -print(f"Resized dimensions in high res mode: {resized_size_high_res}") -tiles_needed = calculate_tiles_needed(*resized_size_high_res) -print(f"Tiles needed for high res image {resized_size_high_res}: {tiles_needed}") diff --git a/tests/local_testing/test_litellm_max_budget.py b/tests/local_testing/test_litellm_max_budget.py deleted file mode 100644 index 9fcddfe32da..00000000000 --- a/tests/local_testing/test_litellm_max_budget.py +++ /dev/null @@ -1,29 +0,0 @@ -# #### What this tests #### -# # This tests calling litellm.max_budget by making back-to-back gpt-4 calls -# # commenting out this test for circle ci, as it causes other tests to fail, since litellm.max_budget would impact other litellm imports -# import sys, os, json -# import traceback -# import pytest - -# sys.path.insert( -# 0, os.path.abspath("../..") -# ) # Adds the parent directory to the system path -# import litellm -# # litellm.set_verbose = True -# from litellm import completion, BudgetExceededError - -# def test_max_budget(): -# try: -# litellm.max_budget = 0.001 # sets a max budget of $0.001 - -# messages = [{"role": "user", "content": "Hey, how's it going"}] -# response = completion(model="gpt-4", messages=messages, stream=True) -# for chunk in response: -# continue -# print(litellm._current_cost) -# completion(model="gpt-4", messages=messages, stream=True) -# litellm.max_budget = float('inf') -# except BudgetExceededError as e: -# pass -# except Exception as e: -# pytest.fail(f"An error occured: {str(e)}") diff --git a/tests/local_testing/test_longer_context_fallback.py b/tests/local_testing/test_longer_context_fallback.py deleted file mode 100644 index adb087079c5..00000000000 --- a/tests/local_testing/test_longer_context_fallback.py +++ /dev/null @@ -1,11 +0,0 @@ -#### What this tests #### -# This tests context fallback dict - -import sys, os -import traceback -import pytest - -import litellm -from litellm import longer_context_model_fallback_dict - -print(longer_context_model_fallback_dict) diff --git a/tests/local_testing/test_mock_request.py b/tests/local_testing/test_mock_request.py deleted file mode 100644 index cd940bc4abb..00000000000 --- a/tests/local_testing/test_mock_request.py +++ /dev/null @@ -1,90 +0,0 @@ -#### What this tests #### -# This tests mock request calls to litellm - -import os - -import pytest - -import litellm -import time - - -def test_mock_request_with_mock_timeout(): - """ - Allow user to set 'mock_timeout = True', this allows for testing if fallbacks/retries are working on timeouts. - """ - start_time = time.time() - with pytest.raises(litellm.Timeout): - response = litellm.completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hey, I'm a mock request"}], - timeout=3, - mock_timeout=True, - ) - end_time = time.time() - assert end_time - start_time >= 3, f"Time taken: {end_time - start_time}" - - -def test_router_mock_request_with_mock_timeout(): - """ - Allow user to set 'mock_timeout = True', this allows for testing if fallbacks/retries are working on timeouts. - """ - start_time = time.time() - router = litellm.Router( - model_list=[ - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, - ], - ) - with pytest.raises(litellm.Timeout): - router.completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hey, I'm a mock request"}], - timeout=3, - mock_timeout=True, - ) - end_time = time.time() - assert end_time - start_time >= 3, f"Time taken: {end_time - start_time}" - - -def test_router_mock_request_with_mock_timeout_with_fallbacks(): - """ - Allow user to set 'mock_timeout = True', this allows for testing if fallbacks/retries are working on timeouts. - """ - litellm.set_verbose = True - start_time = time.time() - router = litellm.Router( - model_list=[ - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, - { - "model_name": "gpt-4.1-nano", - "litellm_params": { - "model": "gpt-4.1-nano", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, - ], - fallbacks=[{"gpt-3.5-turbo": ["gpt-4.1-nano"]}], - ) - response = router.completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hey, I'm a mock request"}], - timeout=3, - num_retries=1, - mock_timeout=True, - ) - print(response) - end_time = time.time() - assert end_time - start_time >= 3, f"Time taken: {end_time - start_time}" - assert "gpt-4.1-nano" in response.model, "Model should be gpt-4.1-nano" diff --git a/tests/local_testing/test_ollama.py b/tests/local_testing/test_ollama.py deleted file mode 100644 index 0a0407e8101..00000000000 --- a/tests/local_testing/test_ollama.py +++ /dev/null @@ -1,125 +0,0 @@ -import json - -from dotenv import load_dotenv - -load_dotenv() - -import pytest - -import litellm - -## for ollama we can't test making the completion call - - -def test_ollama_vision_model(): - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - client = HTTPHandler() - from unittest.mock import patch - - with patch.object(client, "post") as mock_post: - try: - litellm.completion( - model="ollama/llama3.2-vision:11b", - messages=[ - { - "role": "user", - "content": [ - {"type": "text", "text": "Whats in this image?"}, - { - "type": "image_url", - "image_url": { - "url": "https://dummyimage.com/100/100/fff&text=Test+image" - }, - }, - ], - } - ], - client=client, - ) - except Exception as e: - print(e) - mock_post.assert_called() - - print(mock_post.call_args.kwargs) - - json_data = json.loads(mock_post.call_args.kwargs["data"]) - assert json_data["model"] == "llama3.2-vision:11b" - assert "images" in json_data - assert "prompt" in json_data - assert json_data["prompt"].startswith("### User:\n") - - -def test_ollama_ssl_verify(): - import ssl - - import httpx - - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - try: - response = litellm.completion( - model="ollama/llama3.1", - messages=[ - { - "role": "user", - "content": "What's the weather like in San Francisco?", - } - ], - ssl_verify=False, - ) - except Exception as e: - print(e) - - client: HTTPHandler = litellm.in_memory_llm_clients_cache.get_cache( - "httpx_clientssl_verify_False" - ) - - test_client = httpx.Client(verify=False) - print(client) - assert ( - client.client._transport._pool._ssl_context.verify_mode - == test_client._transport._pool._ssl_context.verify_mode - ) - - -@pytest.mark.parametrize("stream", [True, False]) -@pytest.mark.asyncio -async def test_async_ollama_ssl_verify(stream): - import httpx - - from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler - - try: - response = await litellm.acompletion( - model="ollama/llama3.1", - messages=[ - { - "role": "user", - "content": "What's the weather like in San Francisco?", - } - ], - ssl_verify=False, - stream=stream, - ) - except Exception as e: - print(e) - - client: AsyncHTTPHandler = litellm.in_memory_llm_clients_cache.get_cache("async_httpx_clientssl_verify_Falseollama") - - # check client - print("type of transport in client=", type(client.client._transport)) - print("vars in transport in client=", vars(client.client._transport)) - litellm_created_session = client.client._transport.get_valid_client_session() - print("litellm_created_session=", litellm_created_session) - # check session ssl - print("litellm_created_session ssl=", litellm_created_session.connector._ssl) - - # create aiohttp transport with ssl_verify=False - import aiohttp - - aiohttp_session = aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) - print("aiohttp_session ssl=", aiohttp_session.connector._ssl) - - assert litellm_created_session.connector._ssl is False - assert litellm_created_session.connector._ssl == aiohttp_session.connector._ssl diff --git a/tests/local_testing/test_profiling_router.py b/tests/local_testing/test_profiling_router.py deleted file mode 100644 index 5e1646847bc..00000000000 --- a/tests/local_testing/test_profiling_router.py +++ /dev/null @@ -1,152 +0,0 @@ -# #### What this tests #### -# # This profiles a router call to find where calls are taking the most time. - -# import sys, os, time, logging -# import traceback, asyncio, uuid -# import pytest -# import cProfile -# from pstats import Stats -# sys.path.insert( -# 0, os.path.abspath("../..") -# ) # Adds the parent directory to the system path -# import litellm -# from litellm import Router -# from concurrent.futures import ThreadPoolExecutor -# from dotenv import load_dotenv -# from aiodebug import log_slow_callbacks # Import the aiodebug utility for logging slow callbacks - -# # litellm.telemetry = False - -# load_dotenv() - -# logging.basicConfig( -# level=logging.DEBUG, -# format='%(asctime)s %(levelname)s: %(message)s', -# datefmt='%I:%M:%S %p', -# filename='aiologs.log', # Name of the log file where logs will be written -# filemode='w' # 'w' to overwrite the log file on each run, use 'a' to append -# ) - -# # Dictionary to store exception types and their counts -# exception_counts = {} -# exception_data = [] - -# litellm.telemetry = False - -# num_task_cancelled_errors = 0 - -# model_list = [{ -# "model_name": "azure-model", -# "litellm_params": { -# "model": "azure/gpt-turbo", -# "api_key": "os.environ/AZURE_FRANCE_API_KEY", -# "api_base": "https://openai-france-1234.openai.azure.com", -# "rpm": 1440, -# } -# }, { -# "model_name": "azure-model", -# "litellm_params": { -# "model": "azure/gpt-35-turbo", -# "api_key": "os.environ/AZURE_EUROPE_API_KEY", -# "api_base": "https://my-endpoint-europe-berri-992.openai.azure.com", -# "rpm": 6 -# } -# }, { -# "model_name": "azure-model", -# "litellm_params": { -# "model": "azure/gpt-35-turbo", -# "api_key": "os.environ/AZURE_CANADA_API_KEY", -# "api_base": "https://my-endpoint-canada-berri992.openai.azure.com", -# "rpm": 6 -# } -# }] - -# router = Router(model_list=model_list, set_verbose=False, num_retries=3) - -# async def router_completion(): -# global num_task_cancelled_errors, exception_counts -# try: -# messages=[{"role": "user", "content": f"This is a test: {uuid.uuid4()}"}] -# response = await router.acompletion(model="azure-model", messages=messages) -# return response -# except asyncio.exceptions.CancelledError: -# exception_type = "CancelledError" -# exception_counts[exception_type] = exception_counts.get(exception_type, 0) + 1 -# print("Task was cancelled") -# num_task_cancelled_errors += 1 -# exception_data.append({ -# "type": exception_type, -# "traceback": None -# }) -# return None -# except Exception as e: -# exception_type = type(e).__name__ -# exception_counts[exception_type] = exception_counts.get(exception_type, 0) + 1 -# exception_data.append({ -# "type": exception_type, -# "traceback": traceback.format_exc() -# }) -# return None - -# async def loadtest_fn(n = 1452): -# global num_task_cancelled_errors, exception_counts -# start = time.time() -# tasks = [router_completion() for _ in range(n)] -# chat_completions = await asyncio.gather(*tasks) -# successful_completions = [c for c in chat_completions if c is not None] -# print(n, time.time() - start, len(successful_completions)) - -# # Print exception breakdown -# print("Exception Breakdown:") -# for exception_type, count in exception_counts.items(): -# print(f"{exception_type}: {count}") - -# # Store exception_data in a file -# with open('exception_data.txt', 'w') as file: -# for data in exception_data: -# file.write(f"Type: {data['type']}\n") -# if data['traceback']: -# file.write(f"Traceback:\n{data['traceback']}\n\n") - -# loop = asyncio.get_event_loop() -# loop.set_debug(True) -# log_slow_callbacks.enable(0.05) # Log callbacks slower than 0.05 seconds - -# # Excute the load testing function within the asyncio event loop -# loop.run_until_complete(loadtest_fn()) - -# # ### SUSTAINED LOAD TESTS ### -# # import time, asyncio -# # async def make_requests(n): -# # tasks = [router_completion() for _ in range(n)] -# # print(f"num tasks: {len(tasks)}") -# # chat_completions = await asyncio.gather(*tasks) -# # successful_completions = [c for c in chat_completions if c is not None] -# # print(f"successful_completions: {len(successful_completions)}") -# # return successful_completions - -# # async def main(): -# # start_time = time.time() -# # total_successful_requests = 0 -# # request_limit = 1000 -# # batches = 2 # batches of 1k requests -# # start = time.time() -# # tasks = [] # list to hold all tasks - -# # async def request_loop(): -# # nonlocal tasks -# # for _ in range(batches): -# # # Make 1,000 requests -# # task = asyncio.create_task(make_requests(request_limit)) -# # tasks.append(task) - -# # # Introduce a delay to achieve 1,000 requests per second -# # await asyncio.sleep(1) - -# # await request_loop() -# # results = await asyncio.gather(*tasks) -# # total_successful_requests = sum(len(res) for res in results) - -# # print(request_limit*batches, time.time() - start, total_successful_requests) - -# # asyncio.run(main()) diff --git a/tests/local_testing/test_provider_specific_config.py b/tests/local_testing/test_provider_specific_config.py deleted file mode 100644 index 65e75251435..00000000000 --- a/tests/local_testing/test_provider_specific_config.py +++ /dev/null @@ -1,633 +0,0 @@ -#### What this tests #### -# This tests setting provider specific configs across providers -# There are 2 types of tests - changing config dynamically or by setting class variables - -import json -import os -import traceback -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -import litellm -from litellm import RateLimitError, completion - -# Anthropic - - -def claude_test_completion(): - litellm.AnthropicConfig(max_tokens_to_sample=200) - # litellm.set_verbose=True - try: - # OVERRIDE WITH DYNAMIC MAX TOKENS - response_1 = litellm.completion( - model="claude-3-haiku-20240307", - messages=[{"content": "Hello, how are you?", "role": "user"}], - max_tokens=10, - ) - # Add any assertions here to check the response - print(response_1) - response_1_text = response_1.choices[0].message.content - - # USE CONFIG TOKENS - response_2 = litellm.completion( - model="claude-3-haiku-20240307", - messages=[{"content": "Hello, how are you?", "role": "user"}], - ) - # Add any assertions here to check the response - print(response_2) - response_2_text = response_2.choices[0].message.content - - assert len(response_2_text) > len(response_1_text) - - try: - response_3 = litellm.completion( - model="claude-3-5-haiku-20241022", - messages=[{"content": "Hello, how are you?", "role": "user"}], - n=2, - ) - - except Exception as e: - print(e) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# claude_test_completion() - -# Replicate - - -def replicate_test_completion(): - litellm.ReplicateConfig(max_new_tokens=200) - # litellm.set_verbose=True - try: - # OVERRIDE WITH DYNAMIC MAX TOKENS - response_1 = litellm.completion( - model="meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3", - messages=[{"content": "Hello, how are you?", "role": "user"}], - max_tokens=10, - ) - # Add any assertions here to check the response - print(response_1) - response_1_text = response_1.choices[0].message.content - - # USE CONFIG TOKENS - response_2 = litellm.completion( - model="meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3", - messages=[{"content": "Hello, how are you?", "role": "user"}], - ) - # Add any assertions here to check the response - print(response_2) - response_2_text = response_2.choices[0].message.content - - assert len(response_2_text) > len(response_1_text) - try: - response_3 = litellm.completion( - model="meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3", - messages=[{"content": "Hello, how are you?", "role": "user"}], - n=2, - ) - except Exception: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# replicate_test_completion() - -# Cohere - - -def cohere_test_completion(): - # litellm.CohereConfig(max_tokens=200) - litellm.set_verbose = True - try: - # OVERRIDE WITH DYNAMIC MAX TOKENS - response_1 = litellm.completion( - model="command-nightly", - messages=[{"content": "Hello, how are you?", "role": "user"}], - max_tokens=10, - ) - response_1_text = response_1.choices[0].message.content - - # USE CONFIG TOKENS - response_2 = litellm.completion( - model="command-nightly", - messages=[{"content": "Hello, how are you?", "role": "user"}], - ) - response_2_text = response_2.choices[0].message.content - - assert len(response_2_text) > len(response_1_text) - - response_3 = litellm.completion( - model="command-nightly", - messages=[{"content": "Hello, how are you?", "role": "user"}], - n=2, - ) - assert len(response_3.choices) > 1 - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# cohere_test_completion() - - -# ai21_test_completion() - -# TogetherAI - - -def togetherai_test_completion(): - litellm.TogetherAIConfig(max_tokens=10) - litellm.set_verbose = True - try: - # OVERRIDE WITH DYNAMIC MAX TOKENS - response_1 = litellm.completion( - model="together_ai/togethercomputer/llama-2-70b-chat", - messages=[ - { - "content": "Hello, how are you? Be as verbose as possible", - "role": "user", - } - ], - max_tokens=100, - ) - response_1_text = response_1.choices[0].message.content - print(f"response_1_text: {response_1_text}") - - # USE CONFIG TOKENS - response_2 = litellm.completion( - model="together_ai/togethercomputer/llama-2-70b-chat", - messages=[ - { - "content": "Hello, how are you? Be as verbose as possible", - "role": "user", - } - ], - ) - response_2_text = response_2.choices[0].message.content - print(f"response_2_text: {response_2_text}") - - assert len(response_2_text) < len(response_1_text) - - try: - response_3 = litellm.completion( - model="together_ai/togethercomputer/llama-2-70b-chat", - messages=[{"content": "Hello, how are you?", "role": "user"}], - n=2, - ) - pytest.fail(f"Error not raised when n=2 passed to provider") - except Exception: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# togetherai_test_completion() - -# Palm - - -# palm_test_completion() - -# NLP Cloud - - -def nlp_cloud_test_completion(): - litellm.NLPCloudConfig(max_length=10) - # litellm.set_verbose=True - try: - # OVERRIDE WITH DYNAMIC MAX TOKENS - response_1 = litellm.completion( - model="dolphin", - messages=[ - { - "content": "Hello, how are you? Be as verbose as possible", - "role": "user", - } - ], - max_tokens=100, - ) - response_1_text = response_1.choices[0].message.content - print(f"response_1_text: {response_1_text}") - - # USE CONFIG TOKENS - response_2 = litellm.completion( - model="dolphin", - messages=[ - { - "content": "Hello, how are you? Be as verbose as possible", - "role": "user", - } - ], - ) - response_2_text = response_2.choices[0].message.content - print(f"response_2_text: {response_2_text}") - - assert len(response_2_text) < len(response_1_text) - - try: - response_3 = litellm.completion( - model="dolphin", - messages=[{"content": "Hello, how are you?", "role": "user"}], - n=2, - ) - pytest.fail(f"Error not raised when n=2 passed to provider") - except Exception: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# nlp_cloud_test_completion() - -# AlephAlpha - - -def aleph_alpha_test_completion(): - litellm.AlephAlphaConfig(maximum_tokens=10) - # litellm.set_verbose=True - try: - # OVERRIDE WITH DYNAMIC MAX TOKENS - response_1 = litellm.completion( - model="luminous-base", - messages=[ - { - "content": "Hello, how are you? Be as verbose as possible", - "role": "user", - } - ], - max_tokens=100, - ) - response_1_text = response_1.choices[0].message.content - print(f"response_1_text: {response_1_text}") - - # USE CONFIG TOKENS - response_2 = litellm.completion( - model="luminous-base", - messages=[ - { - "content": "Hello, how are you? Be as verbose as possible", - "role": "user", - } - ], - ) - response_2_text = response_2.choices[0].message.content - print(f"response_2_text: {response_2_text}") - - assert len(response_2_text) < len(response_1_text) - - response_3 = litellm.completion( - model="luminous-base", - messages=[{"content": "Hello, how are you?", "role": "user"}], - n=2, - ) - - assert len(response_3.choices) > 1 - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# aleph_alpha_test_completion() - -# Sagemaker - - - - -# sagemaker_test_completion() - - -def test_sagemaker_default_region(): - """ - If no regions are specified in config or in environment, the default region is us-west-2 - """ - mock_response = MagicMock() - - def return_val(): - return { - "generated_text": "This is a mock response from SageMaker.", - "id": "cmpl-mockid", - "object": "text_completion", - "created": 1629800000, - "model": "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", - "choices": [ - { - "text": "This is a mock response from SageMaker.", - "index": 0, - "logprobs": None, - "finish_reason": "length", - } - ], - "usage": {"prompt_tokens": 1, "completion_tokens": 8, "total_tokens": 9}, - } - - mock_response.json = return_val - mock_response.status_code = 200 - - with patch( - "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", - return_value=mock_response, - ) as mock_post: - response = litellm.completion( - model="sagemaker/mock-endpoint", - messages=[{"content": "Hello, world!", "role": "user"}], - ) - mock_post.assert_called_once() - _, kwargs = mock_post.call_args - print(f"kwargs: {kwargs}") - args_to_sagemaker = json.loads(kwargs["data"]) - print("Arguments passed to sagemaker=", args_to_sagemaker) - print("url=", kwargs["url"]) - - assert ( - kwargs["url"] - == "https://runtime.sagemaker.us-west-2.amazonaws.com/endpoints/mock-endpoint/invocations" - ) - - -# test_sagemaker_default_region() - - -def test_sagemaker_environment_region(): - """ - If a region is specified in the environment, use that region instead of us-west-2 - """ - expected_region = "us-east-1" - os.environ["AWS_REGION_NAME"] = expected_region - mock_response = MagicMock() - - def return_val(): - return { - "generated_text": "This is a mock response from SageMaker.", - "id": "cmpl-mockid", - "object": "text_completion", - "created": 1629800000, - "model": "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", - "choices": [ - { - "text": "This is a mock response from SageMaker.", - "index": 0, - "logprobs": None, - "finish_reason": "length", - } - ], - "usage": {"prompt_tokens": 1, "completion_tokens": 8, "total_tokens": 9}, - } - - mock_response.json = return_val - mock_response.status_code = 200 - - with patch( - "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", - return_value=mock_response, - ) as mock_post: - response = litellm.completion( - model="sagemaker/mock-endpoint", - messages=[{"content": "Hello, world!", "role": "user"}], - ) - mock_post.assert_called_once() - _, kwargs = mock_post.call_args - args_to_sagemaker = json.loads(kwargs["data"]) - print("Arguments passed to sagemaker=", args_to_sagemaker) - print("url=", kwargs["url"]) - - assert ( - kwargs["url"] - == f"https://runtime.sagemaker.{expected_region}.amazonaws.com/endpoints/mock-endpoint/invocations" - ) - - del os.environ["AWS_REGION_NAME"] # cleanup - - -# test_sagemaker_environment_region() - - -def test_sagemaker_config_region(): - """ - If a region is specified as part of the optional parameters of the completion, including as - part of the config file, then use that region instead of us-west-2 - """ - expected_region = "us-east-1" - mock_response = MagicMock() - - def return_val(): - return { - "generated_text": "This is a mock response from SageMaker.", - "id": "cmpl-mockid", - "object": "text_completion", - "created": 1629800000, - "model": "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", - "choices": [ - { - "text": "This is a mock response from SageMaker.", - "index": 0, - "logprobs": None, - "finish_reason": "length", - } - ], - "usage": {"prompt_tokens": 1, "completion_tokens": 8, "total_tokens": 9}, - } - - mock_response.json = return_val - mock_response.status_code = 200 - - with patch( - "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", - return_value=mock_response, - ) as mock_post: - - response = litellm.completion( - model="sagemaker/mock-endpoint", - messages=[{"content": "Hello, world!", "role": "user"}], - aws_region_name=expected_region, - ) - - mock_post.assert_called_once() - _, kwargs = mock_post.call_args - args_to_sagemaker = json.loads(kwargs["data"]) - print("Arguments passed to sagemaker=", args_to_sagemaker) - print("url=", kwargs["url"]) - - assert ( - kwargs["url"] - == f"https://runtime.sagemaker.{expected_region}.amazonaws.com/endpoints/mock-endpoint/invocations" - ) - - -# test_sagemaker_config_region() - - -# test_sagemaker_config_and_environment_region() - - -# Bedrock - - -def bedrock_test_completion(): - litellm.AmazonCohereConfig(max_tokens=10) - # litellm.set_verbose=True - try: - # OVERRIDE WITH DYNAMIC MAX TOKENS - response_1 = litellm.completion( - model="bedrock/cohere.command-r-v1:0", - messages=[ - { - "content": "Hello, how are you? Be as verbose as possible", - "role": "user", - } - ], - max_tokens=100, - ) - response_1_text = response_1.choices[0].message.content - print(f"response_1_text: {response_1_text}") - - # USE CONFIG TOKENS - response_2 = litellm.completion( - model="bedrock/cohere.command-r-v1:0", - messages=[ - { - "content": "Hello, how are you? Be as verbose as possible", - "role": "user", - } - ], - ) - response_2_text = response_2.choices[0].message.content - print(f"response_2_text: {response_2_text}") - - assert len(response_2_text) < len(response_1_text) - except RateLimitError: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# bedrock_test_completion() - - -# OpenAI Chat Completion -def openai_test_completion(): - litellm.OpenAIConfig(max_tokens=10) - # litellm.set_verbose=True - try: - # OVERRIDE WITH DYNAMIC MAX TOKENS - response_1 = litellm.completion( - model="gpt-3.5-turbo", - messages=[ - { - "content": "Hello, how are you? Be as verbose as possible", - "role": "user", - } - ], - max_tokens=100, - ) - response_1_text = response_1.choices[0].message.content - print(f"response_1_text: {response_1_text}") - - # USE CONFIG TOKENS - response_2 = litellm.completion( - model="gpt-3.5-turbo", - messages=[ - { - "content": "Hello, how are you? Be as verbose as possible", - "role": "user", - } - ], - ) - response_2_text = response_2.choices[0].message.content - print(f"response_2_text: {response_2_text}") - - assert len(response_2_text) < len(response_1_text) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# openai_test_completion() - - -# OpenAI Text Completion -def openai_text_completion_test(): - litellm.OpenAITextCompletionConfig(max_tokens=10) - # litellm.set_verbose=True - try: - # OVERRIDE WITH DYNAMIC MAX TOKENS - response_1 = litellm.completion( - model="gpt-3.5-turbo-instruct", - messages=[ - { - "content": "Hello, how are you? Be as verbose as possible", - "role": "user", - } - ], - max_tokens=100, - ) - response_1_text = response_1.choices[0].message.content - print(f"response_1_text: {response_1_text}") - - # USE CONFIG TOKENS - response_2 = litellm.completion( - model="gpt-3.5-turbo-instruct", - messages=[ - { - "content": "Hello, how are you? Be as verbose as possible", - "role": "user", - } - ], - ) - response_2_text = response_2.choices[0].message.content - print(f"response_2_text: {response_2_text}") - - assert len(response_2_text) < len(response_1_text) - - response_3 = litellm.completion( - model="gpt-3.5-turbo-instruct", - messages=[{"content": "Hello, how are you?", "role": "user"}], - n=2, - ) - assert len(response_3.choices) > 1 - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# openai_text_completion_test() - - -# Azure OpenAI -def azure_openai_test_completion(): - litellm.AzureOpenAIConfig(max_tokens=10) - # litellm.set_verbose=True - try: - # OVERRIDE WITH DYNAMIC MAX TOKENS - response_1 = litellm.completion( - model="azure/gpt-4.1-mini", - messages=[ - { - "content": "Hello, how are you? Be as verbose as possible", - "role": "user", - } - ], - max_tokens=100, - ) - response_1_text = response_1.choices[0].message.content - print(f"response_1_text: {response_1_text}") - - # USE CONFIG TOKENS - response_2 = litellm.completion( - model="azure/gpt-4.1-mini", - messages=[ - { - "content": "Hello, how are you? Be as verbose as possible", - "role": "user", - } - ], - ) - response_2_text = response_2.choices[0].message.content - print(f"response_2_text: {response_2_text}") - - assert len(response_2_text) < len(response_1_text) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# azure_openai_test_completion() diff --git a/tests/local_testing/test_pydantic.py b/tests/local_testing/test_pydantic.py deleted file mode 100644 index 155b0345186..00000000000 --- a/tests/local_testing/test_pydantic.py +++ /dev/null @@ -1,57 +0,0 @@ -import traceback - -from dotenv import load_dotenv - -load_dotenv() -import io - -import asyncio -import json -import tempfile -from unittest.mock import MagicMock, patch - -import pytest - -import litellm -from litellm.types.utils import ( - ChatCompletionTokenLogprob, - ChoiceLogprobs, - Delta, - ModelResponse, - StreamingChoices, - TopLogprob, -) - -obj = ModelResponse( - id="chat-f9bad6ec3c1146e99368682a0e7403fc", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta(content="", role=None, function_call=None, tool_calls=None), - logprobs=ChoiceLogprobs( - content=[ - ChatCompletionTokenLogprob( - token="", - bytes=[], - logprob=-0.00018153927521780133, - top_logprobs=[ - TopLogprob( - token="", bytes=[], logprob=-0.00018153927521780133 - ), - TopLogprob( - token="\n\n", bytes=[10, 10], logprob=-9.062681198120117 - ), - ], - ) - ] - ), - ) - ], - created=1721976759, - model="Meta-Llama-3-8B-Instruct", - object="chat.completion.chunk", - system_fingerprint=None, -) - -print(obj.model_dump()) diff --git a/tests/local_testing/test_register_model.py b/tests/local_testing/test_register_model.py deleted file mode 100644 index 2ba364aeb03..00000000000 --- a/tests/local_testing/test_register_model.py +++ /dev/null @@ -1,50 +0,0 @@ -#### What this tests #### -# This tests calling batch_completions by running 100 messages together - -import ast -from pathlib import Path - -import pytest - -import litellm - - -# test_update_model_cost_map_url() - - -def test_update_model_cost_via_completion(): - try: - response = litellm.completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hey, how's it going?"}], - input_cost_per_token=0.3, - output_cost_per_token=0.4, - ) - print( - f"litellm.model_cost for gpt-3.5-turbo: {litellm.model_cost['gpt-3.5-turbo']}" - ) - assert litellm.model_cost["gpt-3.5-turbo"]["input_cost_per_token"] == 0.3 - assert litellm.model_cost["gpt-3.5-turbo"]["output_cost_per_token"] == 0.4 - except Exception as e: - pytest.fail(f"An error occurred: {e}") - - -def test_no_test_invocation_at_module_scope(): - tree = ast.parse(Path(__file__).read_text()) - defined = { - node.name - for node in tree.body - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) - } - invoked = [ - node.value.func.id - for node in tree.body - if isinstance(node, ast.Expr) - and isinstance(node.value, ast.Call) - and isinstance(node.value.func, ast.Name) - and node.value.func.id in defined - ] - assert not invoked, ( - f"{invoked} run at import time, so pytest collecting this file fires real " - "provider calls; any failure aborts collection and tears down the whole job" - ) diff --git a/tests/local_testing/test_router_with_fallbacks.py b/tests/local_testing/test_router_with_fallbacks.py deleted file mode 100644 index deabf737501..00000000000 --- a/tests/local_testing/test_router_with_fallbacks.py +++ /dev/null @@ -1,56 +0,0 @@ -# [LOCAL TEST] - runs against mock openai proxy -# # What this tests? -# ## This tests if fallbacks works for 429 errors - -# import sys, os, time -# import traceback, asyncio -# import pytest - -# sys.path.insert( -# 0, os.path.abspath("../..") -# ) # Adds the parent directory to the system path -# import litellm -# from litellm import Router - -# model_list = [ -# { # list of model deployments -# "model_name": "text-embedding-ada-002", # model alias -# "litellm_params": { # params for litellm completion/embedding call -# "model": "text-embedding-ada-002", # actual model name -# "api_key": "sk-fakekey", -# "api_base": "http://0.0.0.0:8080", -# }, -# "tpm": 1000, -# "rpm": 6, -# }, -# { -# "model_name": "text-embedding-ada-002-fallback", -# "litellm_params": { # params for litellm completion/embedding call -# "model": "openai/text-embedding-ada-002-anything-else", # actual model name -# "api_key": "sk-fakekey2", -# "api_base": "http://0.0.0.0:8080", -# }, -# "tpm": 1000, -# "rpm": 6, -# }, -# ] - -# router = Router( -# model_list=model_list, -# fallbacks=[ -# {"text-embedding-ada-002": ["text-embedding-ada-002-fallback"]}, -# {"text-embedding-ada-002-fallback": ["text-embedding-ada-002"]}, -# ], -# set_verbose=True, -# num_retries=0, -# debug_level="INFO", -# routing_strategy="usage-based-routing", -# ) - - -# def test_embedding_with_fallbacks(): -# response = router.embedding(model="text-embedding-ada-002", input=["Hello world"]) -# print(f"response: {response}") - - -# test_embedding_with_fallbacks() diff --git a/tests/local_testing/test_sagemaker.py b/tests/local_testing/test_sagemaker.py deleted file mode 100644 index 8cac665d939..00000000000 --- a/tests/local_testing/test_sagemaker.py +++ /dev/null @@ -1,408 +0,0 @@ -import json - -from dotenv import load_dotenv - -load_dotenv() -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest -from test_streaming import streaming_format_tests - -import litellm -from litellm import completion_cost - -# litellm.num_retries =3 -litellm.cache = None -litellm.success_callback = [] -user_message = "Write a short poem about the sky" -messages = [{"content": user_message, "role": "user"}] -import logging - -from litellm._logging import verbose_logger - - -def logger_fn(user_model_dict): - print(f"user_model_dict: {user_model_dict}") - - -@pytest.fixture(autouse=True) -def reset_callbacks(): - print("\npytest fixture - resetting callbacks") - litellm.success_callback = [] - litellm._async_success_callback = [] - litellm.failure_callback = [] - litellm.callbacks = [] - - -@pytest.mark.asyncio() -@pytest.mark.parametrize("sync_mode", [True, False]) -async def test_completion_sagemaker(sync_mode): - try: - litellm.set_verbose = True - verbose_logger.setLevel(logging.DEBUG) - print("testing sagemaker") - if sync_mode is True: - response = litellm.completion( - model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", - messages=[ - {"role": "user", "content": "hi"}, - ], - temperature=0.2, - max_tokens=80, - cost_per_second=0.000420, - ) - else: - response = await litellm.acompletion( - model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", - messages=[ - {"role": "user", "content": "hi"}, - ], - temperature=0.2, - max_tokens=80, - cost_per_second=0.000420, - ) - # Add any assertions here to check the response - print(response) - cost = completion_cost(completion_response=response) - print("calculated cost", cost) - assert ( - cost > 0.0 and cost < 1.0 - ) # should never be > $1 for a single completion call - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - - - -@pytest.mark.asyncio() -@pytest.mark.parametrize("sync_mode", [False, True]) -@pytest.mark.parametrize( - "model", - [ - # "sagemaker_chat/huggingface-pytorch-tgi-inference-2024-08-23-15-48-59-245", - "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", - ], -) -# @pytest.mark.flaky(retries=3, delay=1) -async def test_completion_sagemaker_stream(sync_mode, model): - try: - litellm.set_verbose = False - print("testing sagemaker") - verbose_logger.setLevel(logging.DEBUG) - full_text = "" - if sync_mode is True: - response = litellm.completion( - model=model, - messages=[ - {"role": "user", "content": "hi - what is ur name"}, - ], - temperature=0.2, - stream=True, - max_tokens=80, - cost_per_second=0.000420, - ) - - for idx, chunk in enumerate(response): - print(chunk) - streaming_format_tests(idx=idx, chunk=chunk) - full_text += chunk.choices[0].delta.content or "" - - print("SYNC RESPONSE full text", full_text) - else: - response = await litellm.acompletion( - model=model, - messages=[ - {"role": "user", "content": "hi - what is ur name"}, - ], - stream=True, - temperature=0.2, - max_tokens=80, - cost_per_second=0.000420, - ) - - print("streaming response") - idx = 0 - async for chunk in response: - print(chunk) - streaming_format_tests(idx=idx, chunk=chunk) - full_text += chunk.choices[0].delta.content or "" - idx += 1 - - print("ASYNC RESPONSE full text", full_text) - - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -@pytest.mark.asyncio() -@pytest.mark.parametrize("sync_mode", [False, True]) -@pytest.mark.parametrize( - "model", - [ - # "sagemaker_chat/huggingface-pytorch-tgi-inference-2024-08-23-15-48-59-245", - "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", - ], -) -async def test_completion_sagemaker_streaming_bad_request(sync_mode, model): - litellm.set_verbose = True - print("testing sagemaker") - if sync_mode is True: - with pytest.raises(litellm.BadRequestError): - response = litellm.completion( - model=model, - messages=[ - {"role": "user", "content": "hi"}, - ], - stream=True, - max_tokens=8000000000000000, - ) - else: - with pytest.raises(litellm.BadRequestError): - response = await litellm.acompletion( - model=model, - messages=[ - {"role": "user", "content": "hi"}, - ], - stream=True, - max_tokens=8000000000000000, - ) - - -@pytest.mark.asyncio -async def test_acompletion_sagemaker_non_stream(): - mock_response = AsyncMock() - - def return_val(): - return { - "generated_text": "This is a mock response from SageMaker.", - "id": "cmpl-mockid", - "object": "text_completion", - "created": 1629800000, - "model": "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", - "choices": [ - { - "text": "This is a mock response from SageMaker.", - "index": 0, - "logprobs": None, - "finish_reason": "length", - } - ], - "usage": {"prompt_tokens": 1, "completion_tokens": 8, "total_tokens": 9}, - } - - mock_response.json = return_val - mock_response.status_code = 200 - - expected_payload = { - "inputs": "hi", - "parameters": {"temperature": 0.2, "max_new_tokens": 80}, - } - - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - return_value=mock_response, - ) as mock_post: - # Act: Call the litellm.acompletion function - response = await litellm.acompletion( - model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", - messages=[ - {"role": "user", "content": "hi"}, - ], - temperature=0.2, - max_tokens=80, - cost_per_second=0.000420, - ) - - # Print what was called on the mock - print("call args=", mock_post.call_args) - - # Assert - mock_post.assert_called_once() - _, kwargs = mock_post.call_args - args_to_sagemaker = json.loads(kwargs["data"]) - print("Arguments passed to sagemaker=", args_to_sagemaker) - assert args_to_sagemaker == expected_payload - assert ( - kwargs["url"] - == "https://runtime.sagemaker.us-west-2.amazonaws.com/endpoints/jumpstart-dft-hf-textgeneration1-mp-20240815-185614/invocations" - ) - - -@pytest.mark.asyncio -async def test_completion_sagemaker_non_stream(): - mock_response = MagicMock() - - def return_val(): - return { - "generated_text": "This is a mock response from SageMaker.", - "id": "cmpl-mockid", - "object": "text_completion", - "created": 1629800000, - "model": "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", - "choices": [ - { - "text": "This is a mock response from SageMaker.", - "index": 0, - "logprobs": None, - "finish_reason": "length", - } - ], - "usage": {"prompt_tokens": 1, "completion_tokens": 8, "total_tokens": 9}, - } - - mock_response.json = return_val - mock_response.status_code = 200 - - expected_payload = { - "inputs": "hi", - "parameters": {"temperature": 0.2, "max_new_tokens": 80}, - } - - with patch( - "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", - return_value=mock_response, - ) as mock_post: - # Act: Call the litellm.acompletion function - response = litellm.completion( - model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", - messages=[ - {"role": "user", "content": "hi"}, - ], - temperature=0.2, - max_tokens=80, - cost_per_second=0.000420, - ) - - # Print what was called on the mock - print("call args=", mock_post.call_args) - - # Assert - mock_post.assert_called_once() - _, kwargs = mock_post.call_args - args_to_sagemaker = json.loads(kwargs["data"]) - print("Arguments passed to sagemaker=", args_to_sagemaker) - assert args_to_sagemaker == expected_payload - assert ( - kwargs["url"] - == "https://runtime.sagemaker.us-west-2.amazonaws.com/endpoints/jumpstart-dft-hf-textgeneration1-mp-20240815-185614/invocations" - ) - - -@pytest.mark.asyncio -@pytest.mark.flaky(retries=3, delay=1) -async def test_completion_sagemaker_prompt_template_non_stream(): - mock_response = MagicMock() - - def return_val(): - return { - "generated_text": "This is a mock response from SageMaker.", - "id": "cmpl-mockid", - "object": "text_completion", - "created": 1629800000, - "model": "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", - "choices": [ - { - "text": "This is a mock response from SageMaker.", - "index": 0, - "logprobs": None, - "finish_reason": "length", - } - ], - "usage": {"prompt_tokens": 1, "completion_tokens": 8, "total_tokens": 9}, - } - - mock_response.json = return_val - mock_response.status_code = 200 - - expected_payload = { - "inputs": "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n\n### Instruction:\nhi\n\n\n### Response:\n", - "parameters": {"temperature": 0.2, "max_new_tokens": 80}, - } - - with patch( - "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", - return_value=mock_response, - ) as mock_post: - # Act: Call the litellm.acompletion function - response = litellm.completion( - model="sagemaker/deepseek_coder_6.7_instruct", - messages=[ - {"role": "user", "content": "hi"}, - ], - temperature=0.2, - max_tokens=80, - hf_model_name="deepseek-ai/deepseek-coder-6.7b-instruct", - ) - - # Print what was called on the mock - print("call args=", mock_post.call_args) - - # Assert - mock_post.assert_called_once() - _, kwargs = mock_post.call_args - args_to_sagemaker = json.loads(kwargs["data"]) - print("Arguments passed to sagemaker=", args_to_sagemaker) - assert args_to_sagemaker == expected_payload - - -@pytest.mark.asyncio -async def test_completion_sagemaker_non_stream_with_aws_params(): - mock_response = MagicMock() - - def return_val(): - return { - "generated_text": "This is a mock response from SageMaker.", - "id": "cmpl-mockid", - "object": "text_completion", - "created": 1629800000, - "model": "sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", - "choices": [ - { - "text": "This is a mock response from SageMaker.", - "index": 0, - "logprobs": None, - "finish_reason": "length", - } - ], - "usage": {"prompt_tokens": 1, "completion_tokens": 8, "total_tokens": 9}, - } - - mock_response.json = return_val - mock_response.status_code = 200 - - expected_payload = { - "inputs": "hi", - "parameters": {"temperature": 0.2, "max_new_tokens": 80}, - } - - with patch( - "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", - return_value=mock_response, - ) as mock_post: - # Act: Call the litellm.acompletion function - response = litellm.completion( - model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", - messages=[ - {"role": "user", "content": "hi"}, - ], - temperature=0.2, - max_tokens=80, - cost_per_second=0.000420, - aws_access_key_id="gm", - aws_secret_access_key="s", - aws_region_name="us-west-5", - ) - - # Print what was called on the mock - print("call args=", mock_post.call_args) - - # Assert - mock_post.assert_called_once() - _, kwargs = mock_post.call_args - args_to_sagemaker = json.loads(kwargs["data"]) - print("Arguments passed to sagemaker=", args_to_sagemaker) - assert args_to_sagemaker == expected_payload - assert ( - kwargs["url"] - == "https://runtime.sagemaker.us-west-5.amazonaws.com/endpoints/jumpstart-dft-hf-textgeneration1-mp-20240815-185614/invocations" - ) diff --git a/tests/local_testing/test_sagemaker_nova_integration.py b/tests/local_testing/test_sagemaker_nova_integration.py deleted file mode 100644 index 95f28fe9892..00000000000 --- a/tests/local_testing/test_sagemaker_nova_integration.py +++ /dev/null @@ -1,278 +0,0 @@ -""" -Integration tests for SageMaker Nova provider. - -These tests require a live SageMaker Nova endpoint and AWS credentials. -They are skipped by default — run manually with: - - pytest tests/local_testing/test_sagemaker_nova_integration.py -v --no-header -rN - -Prerequisites: - export AWS_PROFILE= # or set AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY - export AWS_REGION_NAME=us-east-1 - export SAGEMAKER_NOVA_ENDPOINT= -""" - -import base64 -import io -import json -import os -import struct -import zlib - -import pytest - -import litellm - -ENDPOINT = os.environ.get("SAGEMAKER_NOVA_ENDPOINT", "") -MODEL = f"sagemaker_nova/{ENDPOINT}" - -skip_if_no_endpoint = pytest.mark.skipif( - not ENDPOINT, - reason="SAGEMAKER_NOVA_ENDPOINT not set — skipping live integration tests", -) - - -def _make_test_png() -> str: - """Create a minimal 4x4 PNG (red border, blue center) and return base64.""" - - def chunk(ctype, data): - c = ctype + data - return ( - struct.pack(">I", len(data)) - + c - + struct.pack(">I", zlib.crc32(c) & 0xFFFFFFFF) - ) - - width, height = 4, 4 - pixels = [] - for y in range(height): - for x in range(width): - if 1 <= x <= 2 and 1 <= y <= 2: - pixels.append((0, 0, 255)) - else: - pixels.append((255, 0, 0)) - - raw = b"" - for y in range(height): - raw += b"\x00" - for x in range(width): - raw += bytes(pixels[y * width + x]) - - png = ( - b"\x89PNG\r\n\x1a\n" - + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) - + chunk(b"IDAT", zlib.compress(raw)) - + chunk(b"IEND", b"") - ) - return base64.b64encode(png).decode() - - -@skip_if_no_endpoint -class TestSagemakerNovaIntegration: - """Live integration tests for sagemaker_nova provider.""" - - def test_should_complete_basic_single_turn(self): - """Basic single-turn chat completion.""" - response = litellm.completion( - model=MODEL, - messages=[{"role": "user", "content": "What is 2+2? Reply in one word."}], - max_tokens=32, - temperature=0.1, - ) - assert response.choices[0].message.content is not None - assert len(response.choices[0].message.content.strip()) > 0 - assert response.choices[0].finish_reason == "stop" - assert response.usage.prompt_tokens > 0 - assert response.usage.completion_tokens > 0 - assert response.usage.total_tokens == ( - response.usage.prompt_tokens + response.usage.completion_tokens - ) - - def test_should_complete_multi_turn_conversation(self): - """Multi-turn conversation maintains context.""" - messages = [ - {"role": "user", "content": "My name is Alice."}, - ] - response1 = litellm.completion( - model=MODEL, - messages=messages, - max_tokens=64, - temperature=0.1, - ) - assistant_msg = response1.choices[0].message.content - assert assistant_msg is not None - - # Second turn — model should remember the name - messages.append({"role": "assistant", "content": assistant_msg}) - messages.append({"role": "user", "content": "What is my name?"}) - - response2 = litellm.completion( - model=MODEL, - messages=messages, - max_tokens=64, - temperature=0.1, - ) - answer = response2.choices[0].message.content.lower() - assert "alice" in answer, f"Expected 'alice' in response, got: {answer}" - - def test_should_stream_response(self): - """Streaming returns chunks with content and final usage.""" - response = litellm.completion( - model=MODEL, - messages=[{"role": "user", "content": "Count from 1 to 5."}], - max_tokens=64, - stream=True, - stream_options={"include_usage": True}, - ) - - chunks = [] - full_content = "" - for chunk in response: - chunks.append(chunk) - delta = chunk.choices[0].delta.content or "" - full_content += delta - - assert len(chunks) > 1, "Expected multiple streaming chunks" - assert len(full_content.strip()) > 0, "Expected non-empty streamed content" - - # Last chunk should have finish_reason - final_chunks_with_finish = [ - c for c in chunks if c.choices and c.choices[0].finish_reason is not None - ] - assert ( - len(final_chunks_with_finish) > 0 - ), "Expected at least one chunk with finish_reason" - - def test_should_return_logprobs(self): - """Logprobs are returned when requested.""" - response = litellm.completion( - model=MODEL, - messages=[{"role": "user", "content": "Say hello."}], - max_tokens=16, - temperature=0.1, - logprobs=True, - top_logprobs=3, - ) - lp = response.choices[0].logprobs - assert lp is not None, "Expected logprobs in response" - - content = lp.content if hasattr(lp, "content") else lp.get("content") - assert content is not None and len(content) > 0, "Expected logprobs content" - - first_token = content[0] - assert "token" in first_token or hasattr(first_token, "token") - assert "logprob" in first_token or hasattr(first_token, "logprob") - - top = ( - first_token.get("top_logprobs") - if isinstance(first_token, dict) - else first_token.top_logprobs - ) - assert top is not None and len(top) == 3, "Expected 3 top_logprobs" - - def test_should_handle_multimodal_image_input(self): - """Multimodal with base64 image in content array.""" - b64_image = _make_test_png() - response = litellm.completion( - model=MODEL, - messages=[ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "What colors do you see in this image? List them.", - }, - { - "type": "image_url", - "image_url": {"url": f"data:image/png;base64,{b64_image}"}, - }, - ], - } - ], - max_tokens=128, - ) - content = response.choices[0].message.content.lower() - assert response.choices[0].message.content is not None - assert len(content) > 0 - # The image has red and blue — model should mention at least one - assert ( - "red" in content or "blue" in content - ), f"Expected 'red' or 'blue' in multimodal response, got: {content}" - - def test_should_pass_nova_specific_params(self): - """Nova-specific parameters (top_k) are accepted.""" - response = litellm.completion( - model=MODEL, - messages=[{"role": "user", "content": "Say hello."}], - max_tokens=32, - top_k=40, - temperature=0.7, - ) - assert response.choices[0].message.content is not None - assert response.usage.total_tokens > 0 - - def test_should_respect_system_message(self): - """System message should influence the response.""" - response = litellm.completion( - model=MODEL, - messages=[ - { - "role": "system", - "content": "You are a pirate. Always respond in pirate speak.", - }, - {"role": "user", "content": "How are you today?"}, - ], - max_tokens=128, - temperature=0.7, - ) - content = response.choices[0].message.content.lower() - assert response.choices[0].message.content is not None - # Pirate-themed words likely in response - pirate_words = ["arr", "ahoy", "matey", "ye", "sail", "sea", "cap"] - assert any( - w in content for w in pirate_words - ), f"Expected pirate speak, got: {content}" - - -NOVA2_ENDPOINT = os.environ.get("SAGEMAKER_NOVA2_LITE_ENDPOINT", "") -NOVA2_MODEL = f"sagemaker_nova/{NOVA2_ENDPOINT}" - -skip_if_no_nova2_endpoint = pytest.mark.skipif( - not NOVA2_ENDPOINT, - reason="SAGEMAKER_NOVA2_LITE_ENDPOINT not set — requires Nova 2 Lite endpoint", -) - - -@skip_if_no_nova2_endpoint -class TestSagemakerNova2LiteIntegration: - """ - Integration tests requiring a Nova 2 Lite endpoint (reasoning_effort support). - - Run with: - export SAGEMAKER_NOVA2_LITE_ENDPOINT= - pytest tests/local_testing/test_sagemaker_nova_integration.py::TestSagemakerNova2LiteIntegration -v - """ - - def test_should_accept_reasoning_effort_low(self): - """reasoning_effort='low' should be accepted by Nova 2 Lite.""" - response = litellm.completion( - model=NOVA2_MODEL, - messages=[{"role": "user", "content": "What is 2+2?"}], - max_tokens=32, - reasoning_effort="low", - ) - assert response.choices[0].message.content is not None - assert response.usage.total_tokens > 0 - - def test_should_accept_reasoning_effort_high(self): - """reasoning_effort='high' should be accepted by Nova 2 Lite.""" - response = litellm.completion( - model=NOVA2_MODEL, - messages=[{"role": "user", "content": "Explain why the sky is blue."}], - max_tokens=256, - reasoning_effort="high", - ) - assert response.choices[0].message.content is not None - assert len(response.choices[0].message.content) > 0 - assert response.usage.completion_tokens > 0 diff --git a/tests/local_testing/test_stream_chunk_builder.py b/tests/local_testing/test_stream_chunk_builder.py deleted file mode 100644 index 6d62dd52b89..00000000000 --- a/tests/local_testing/test_stream_chunk_builder.py +++ /dev/null @@ -1,897 +0,0 @@ -import asyncio -import os -import time -import traceback - -import pytest -from typing import List -from litellm.types.utils import StreamingChoices, ChatCompletionAudioResponse - - -def check_non_streaming_response(response): - assert response.choices[0].message.audio is not None, "Audio response is missing" - print("audio", response.choices[0].message.audio) - assert isinstance( - response.choices[0].message.audio, ChatCompletionAudioResponse - ), "Invalid audio response type" - assert len(response.choices[0].message.audio.data) > 0, "Audio data is empty" - - - -import dotenv -from openai import OpenAI - -import litellm -import stream_chunk_testdata -from litellm import completion, stream_chunk_builder - -dotenv.load_dotenv() - -user_message = "What is the current weather in Boston?" -messages = [{"content": user_message, "role": "user"}] - -function_schema = { - "name": "get_weather", - "description": "gets the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - }, - "required": ["location"], - }, -} - - -tools_schema = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - }, - } -] - -# def test_stream_chunk_builder_tools(): -# try: -# litellm.set_verbose = False -# response = client.chat.completions.create( -# model="gpt-3.5-turbo", -# messages=messages, -# tools=tools_schema, -# # stream=True, -# # complete_response=True # runs stream_chunk_builder under-the-hood -# ) - -# print(f"response: {response}") -# print(f"response usage: {response.usage}") -# except Exception as e: -# pytest.fail(f"An exception occurred - {str(e)}") - -# test_stream_chunk_builder_tools() - - -def test_stream_chunk_builder_litellm_function_call(): - try: - litellm.set_verbose = False - response = litellm.completion( - model="gpt-3.5-turbo", - messages=messages, - functions=[function_schema], - # stream=True, - # complete_response=True # runs stream_chunk_builder under-the-hood - ) - - print(f"response: {response}") - except Exception as e: - pytest.fail(f"An exception occurred - {str(e)}") - - -# test_stream_chunk_builder_litellm_function_call() - - -def test_stream_chunk_builder_litellm_tool_call(): - try: - litellm.set_verbose = True - response = litellm.completion( - model="gpt-3.5-turbo", - messages=messages, - tools=tools_schema, - stream=True, - complete_response=True, - ) - - print(f"complete response: {response}") - print(f"complete response usage: {response.usage}") - assert response.usage.completion_tokens > 0 - assert response.usage.prompt_tokens > 0 - assert ( - response.usage.total_tokens - == response.usage.completion_tokens + response.usage.prompt_tokens - ) - except Exception as e: - pytest.fail(f"An exception occurred - {str(e)}") - - -# test_stream_chunk_builder_litellm_tool_call() - - -def test_stream_chunk_builder_litellm_tool_call_regular_message(): - try: - messages = [{"role": "user", "content": "Hey, how's it going?"}] - # litellm.set_verbose = True - response = litellm.completion( - model="gpt-3.5-turbo", - messages=messages, - tools=tools_schema, - stream=True, - complete_response=True, - ) - - print(f"complete response: {response}") - print(f"complete response usage: {response.usage}") - assert response.usage.completion_tokens > 0 - assert response.usage.prompt_tokens > 0 - assert ( - response.usage.total_tokens - == response.usage.completion_tokens + response.usage.prompt_tokens - ) - - # check provider is in hidden params - print("hidden params", response._hidden_params) - assert response._hidden_params["custom_llm_provider"] == "openai" - - except Exception as e: - pytest.fail(f"An exception occurred - {str(e)}") - - -# test_stream_chunk_builder_litellm_tool_call_regular_message() - - -def test_stream_chunk_builder_litellm_mixed_calls(): - response = stream_chunk_builder(stream_chunk_testdata.chunks) - assert ( - response.choices[0].message.content - == "To answer your question about how many rows are in the 'users' table, I'll need to run a SQL query. Let me do that for you." - ) - - print(response.choices[0].message.tool_calls[0].to_dict()) - - assert len(response.choices[0].message.tool_calls) == 1 - assert response.choices[0].message.tool_calls[0].to_dict() == { - "function": { - "arguments": '{"query": "SELECT COUNT(*) FROM users;"}', - "name": "sql_query", - }, - "id": "toolu_01H3AjkLpRtGQrof13CBnWfK", - "type": "function", - } - - -def test_stream_chunk_builder_litellm_empty_chunks(): - with pytest.raises(litellm.APIError): - response = stream_chunk_builder(chunks=None) - - response = stream_chunk_builder(chunks=[]) - assert response is None - - -def test_stream_chunk_builder_multiple_tool_calls(): - init_chunks = [ - { - "id": "chatcmpl-A5kCnzaxRsknd6008552ZhDi71yPt", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "tool_calls": [ - { - "id": "call_X9P9B6STj7ze8OsJCGkfoN94", - "function": {"arguments": "", "name": "exponentiate"}, - "type": "function", - "index": 0, - } - ], - }, - } - ], - "created": 1725932618, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion.chunk", - "system_fingerprint": "fp_b2ffeb16ee", - }, - { - "id": "chatcmpl-A5kCnzaxRsknd6008552ZhDi71yPt", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "tool_calls": [ - { - "function": {"arguments": '{"ba'}, - "type": "function", - "index": 0, - } - ], - }, - } - ], - "created": 1725932618, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion.chunk", - "system_fingerprint": "fp_b2ffeb16ee", - }, - { - "id": "chatcmpl-A5kCnzaxRsknd6008552ZhDi71yPt", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "tool_calls": [ - { - "function": {"arguments": 'se": '}, - "type": "function", - "index": 0, - } - ], - }, - } - ], - "created": 1725932618, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion.chunk", - "system_fingerprint": "fp_b2ffeb16ee", - }, - { - "id": "chatcmpl-A5kCnzaxRsknd6008552ZhDi71yPt", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "tool_calls": [ - { - "function": {"arguments": '3, "ex'}, - "type": "function", - "index": 0, - } - ], - }, - } - ], - "created": 1725932618, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion.chunk", - "system_fingerprint": "fp_b2ffeb16ee", - }, - { - "id": "chatcmpl-A5kCnzaxRsknd6008552ZhDi71yPt", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "tool_calls": [ - { - "function": {"arguments": "pone"}, - "type": "function", - "index": 0, - } - ], - }, - } - ], - "created": 1725932618, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion.chunk", - "system_fingerprint": "fp_b2ffeb16ee", - }, - { - "id": "chatcmpl-A5kCnzaxRsknd6008552ZhDi71yPt", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "tool_calls": [ - { - "function": {"arguments": 'nt": '}, - "type": "function", - "index": 0, - } - ], - }, - } - ], - "created": 1725932618, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion.chunk", - "system_fingerprint": "fp_b2ffeb16ee", - }, - { - "id": "chatcmpl-A5kCnzaxRsknd6008552ZhDi71yPt", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "tool_calls": [ - { - "function": {"arguments": "5}"}, - "type": "function", - "index": 0, - } - ], - }, - } - ], - "created": 1725932618, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion.chunk", - "system_fingerprint": "fp_b2ffeb16ee", - }, - { - "id": "chatcmpl-A5kCnzaxRsknd6008552ZhDi71yPt", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "tool_calls": [ - { - "id": "call_Qq8yDeRx7v276abRcLrYORdW", - "function": {"arguments": "", "name": "add"}, - "type": "function", - "index": 1, - } - ], - }, - } - ], - "created": 1725932618, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion.chunk", - "system_fingerprint": "fp_b2ffeb16ee", - }, - { - "id": "chatcmpl-A5kCnzaxRsknd6008552ZhDi71yPt", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "tool_calls": [ - { - "function": {"arguments": '{"fi'}, - "type": "function", - "index": 1, - } - ], - }, - } - ], - "created": 1725932618, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion.chunk", - "system_fingerprint": "fp_b2ffeb16ee", - }, - { - "id": "chatcmpl-A5kCnzaxRsknd6008552ZhDi71yPt", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "tool_calls": [ - { - "function": {"arguments": "rst_i"}, - "type": "function", - "index": 1, - } - ], - }, - } - ], - "created": 1725932618, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion.chunk", - "system_fingerprint": "fp_b2ffeb16ee", - }, - { - "id": "chatcmpl-A5kCnzaxRsknd6008552ZhDi71yPt", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "tool_calls": [ - { - "function": {"arguments": 'nt": 1'}, - "type": "function", - "index": 1, - } - ], - }, - } - ], - "created": 1725932618, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion.chunk", - "system_fingerprint": "fp_b2ffeb16ee", - }, - { - "id": "chatcmpl-A5kCnzaxRsknd6008552ZhDi71yPt", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "tool_calls": [ - { - "function": {"arguments": '2, "'}, - "type": "function", - "index": 1, - } - ], - }, - } - ], - "created": 1725932618, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion.chunk", - "system_fingerprint": "fp_b2ffeb16ee", - }, - { - "id": "chatcmpl-A5kCnzaxRsknd6008552ZhDi71yPt", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "tool_calls": [ - { - "function": {"arguments": "secon"}, - "type": "function", - "index": 1, - } - ], - }, - } - ], - "created": 1725932618, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion.chunk", - "system_fingerprint": "fp_b2ffeb16ee", - }, - { - "id": "chatcmpl-A5kCnzaxRsknd6008552ZhDi71yPt", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "tool_calls": [ - { - "function": {"arguments": 'd_int"'}, - "type": "function", - "index": 1, - } - ], - }, - } - ], - "created": 1725932618, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion.chunk", - "system_fingerprint": "fp_b2ffeb16ee", - }, - { - "id": "chatcmpl-A5kCnzaxRsknd6008552ZhDi71yPt", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "tool_calls": [ - { - "function": {"arguments": ": 3}"}, - "type": "function", - "index": 1, - } - ], - }, - } - ], - "created": 1725932618, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion.chunk", - "system_fingerprint": "fp_b2ffeb16ee", - }, - { - "id": "chatcmpl-A5kCnzaxRsknd6008552ZhDi71yPt", - "choices": [{"finish_reason": "tool_calls", "index": 0, "delta": {}}], - "created": 1725932618, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion.chunk", - "system_fingerprint": "fp_b2ffeb16ee", - }, - ] - - chunks = [] - for chunk in init_chunks: - chunks.append(litellm.ModelResponseStream(**chunk)) - response = stream_chunk_builder(chunks=chunks) - - print(f"Returned response: {response}") - completed_response = { - "id": "chatcmpl-A61mXjvcRX0Xr2IiojN9TPiy1P3Fm", - "choices": [ - { - "finish_reason": "tool_calls", - "index": 0, - "message": { - "content": None, - "role": "assistant", - "tool_calls": [ - { - "function": { - "arguments": '{"base": 3, "exponent": 5}', - "name": "exponentiate", - }, - "id": "call_X9P9B6STj7ze8OsJCGkfoN94", - "type": "function", - }, - { - "function": { - "arguments": '{"first_int": 12, "second_int": 3}', - "name": "add", - }, - "id": "call_Qq8yDeRx7v276abRcLrYORdW", - "type": "function", - }, - ], - "function_call": None, - }, - } - ], - "created": 1726000181, - "model": "gpt-4o-2024-05-13", - "object": "chat.completion", - "system_fingerprint": "fp_25624ae3a5", - "usage": {"completion_tokens": 55, "prompt_tokens": 127, "total_tokens": 182}, - "service_tier": None, - } - - expected_response = litellm.ModelResponse(**completed_response) - - print(f"\n\nexpected_response:\n{expected_response}\n\n") - assert ( - expected_response.choices == response.choices - ), "\nGot={}\n, Expected={}\n".format(response.choices, expected_response.choices) - - -def test_stream_chunk_builder_openai_prompt_caching(): - from pydantic import BaseModel - - client = OpenAI( - # This is the default and can be omitted - api_key=os.getenv("OPENAI_API_KEY"), - ) - - chat_completion = client.chat.completions.create( - messages=[ - { - "role": "user", - "content": "Say this is a test", - } - ], - model="gpt-3.5-turbo", - stream=True, - stream_options={"include_usage": True}, - ) - chunks: List[litellm.ModelResponse] = [] - usage_obj = None - for chunk in chat_completion: - chunks.append(litellm.ModelResponseStream(**chunk.model_dump())) - - print(f"chunks: {chunks}") - - usage_obj: litellm.Usage = chunks[-1].usage # type: ignore - - response = stream_chunk_builder(chunks=chunks) - print(f"response: {response}") - print(f"response usage: {response.usage}") - for k, v in usage_obj.model_dump(exclude_none=True).items(): - print(k, v) - response_usage_value = getattr(response.usage, k) # type: ignore - print(f"response_usage_value: {response_usage_value}") - print(f"type: {type(response_usage_value)}") - if isinstance(response_usage_value, BaseModel): - assert response_usage_value.model_dump(exclude_none=True) == v - else: - assert response_usage_value == v - - -@pytest.mark.flaky(retries=5, delay=2) -def test_stream_chunk_builder_openai_audio_output_usage(): - from pydantic import BaseModel - from typing import Optional - - client = OpenAI( - # This is the default and can be omitted - api_key=os.getenv("OPENAI_API_KEY"), - ) - - try: - completion = client.chat.completions.create( - model="gpt-audio-1.5", - modalities=["text", "audio"], - audio={"voice": "alloy", "format": "pcm16"}, - messages=[{"role": "user", "content": "response in 1 word - yes or no"}], - stream=True, - stream_options={"include_usage": True}, - ) - except Exception as e: - err = str(e).lower() - if ( - "model_not_found" in err - or "does not exist" in err - or "openai-internal" in err - ): - pytest.skip(f"Skipping - upstream gpt-audio-1.5 unavailable: {e}") - raise - - chunks = [] - for chunk in completion: - chunks.append(litellm.ModelResponseStream(**chunk.model_dump())) - - usage_obj: Optional[litellm.Usage] = None - - for index, chunk in enumerate(chunks): - if hasattr(chunk, "usage") and chunk.usage is not None: - usage_obj = chunk.usage - print(f"chunk usage: {chunk.usage}") - print(f"index: {index}") - print(f"len chunks: {len(chunks)}") - - print(f"usage_obj: {usage_obj}") - if usage_obj is None: - pytest.skip("OpenAI did not return usage data in streaming response") - response = stream_chunk_builder(chunks=chunks) - print(f"response usage: {response.usage}") - check_non_streaming_response(response) - print(f"response: {response}") - # Convert both usage objects to dictionaries for easier comparison - usage_dict = usage_obj.model_dump(exclude_none=True) - response_usage_dict = response.usage.model_dump(exclude_none=True) - - # Simple dictionary comparison - assert ( - usage_dict == response_usage_dict - ), f"\nExpected: {usage_dict}\nGot: {response_usage_dict}" - - -def test_stream_chunk_builder_empty_initial_chunk(): - from litellm.litellm_core_utils.streaming_chunk_builder_utils import ( - ChunkProcessor, - ) - - chunks = [ - {"id": ""}, - {"id": "1"}, - {"id": "1"}, - ] - - id = ChunkProcessor._get_chunk_id(chunks) - assert id == "1" - - -def test_stream_chunk_builder_tool_calls_list(): - from litellm.litellm_core_utils.streaming_chunk_builder_utils import ( - ChunkProcessor, - ) - from litellm.types.utils import ( - ChatCompletionMessageToolCall, - Function, - ModelResponseStream, - Delta, - ChatCompletionDeltaToolCall, - ) - - chunks = [ - ModelResponseStream( - id="chatcmpl-f323f7a5-2da0-4f86-8ed7-c653c5a359d9", - created=1736388417, - model="llama-3.3-70b-versatile", - object="chat.completion.chunk", - system_fingerprint=None, - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="", - role="assistant", - function_call=None, - tool_calls=[ - ChatCompletionDeltaToolCall( - id="call_9y79", - function=Function( - arguments='{"location": "San Francisco", "unit": "celsius"}', - name="get_current_weather", - ), - type="function", - index=0, - ) - ], - audio=None, - ), - logprobs=None, - ) - ], - stream_options=None, - ), - ModelResponseStream( - id="chatcmpl-f323f7a5-2da0-4f86-8ed7-c653c5a359d9", - created=1736388417, - model="llama-3.3-70b-versatile", - object="chat.completion.chunk", - system_fingerprint=None, - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="", - role=None, - function_call=None, - tool_calls=[ - ChatCompletionDeltaToolCall( - id="call_pfp7", - function=Function( - arguments='{"location": "Tokyo", "unit": "celsius"}', - name="get_current_weather", - ), - type="function", - index=1, - ) - ], - audio=None, - ), - logprobs=None, - ) - ], - stream_options=None, - ), - ModelResponseStream( - id="chatcmpl-f323f7a5-2da0-4f86-8ed7-c653c5a359d9", - created=1736388417, - model="llama-3.3-70b-versatile", - object="chat.completion.chunk", - system_fingerprint=None, - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="", - role=None, - function_call=None, - tool_calls=[ - ChatCompletionDeltaToolCall( - id="call_hyj5", - function=Function( - arguments='{"location": "Paris", "unit": "celsius"}', - name="get_current_weather", - ), - type="function", - index=2, - ) - ], - audio=None, - ), - logprobs=None, - ) - ], - stream_options=None, - ), - ] - - processor = ChunkProcessor(chunks=chunks) - - tool_calls = processor.get_combined_tool_content(tool_call_chunks=chunks) - print(f"tool_calls: {tool_calls}") - assert len(tool_calls) == 3 - - -import json - - -def get_current_weather(location, unit="fahrenheit"): - """Get the current weather in a given location""" - if "tokyo" in location.lower(): - return json.dumps({"location": "Tokyo", "temperature": "10", "unit": "celsius"}) - elif "san francisco" in location.lower(): - return json.dumps( - {"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"} - ) - elif "paris" in location.lower(): - return json.dumps({"location": "Paris", "temperature": "22", "unit": "celsius"}) - else: - return json.dumps({"location": location, "temperature": "unknown"}) - - -@pytest.fixture(scope="module", autouse=True) -def load_env(): - messages = [ - {"role": "system", "content": "You are a helpful AI assistant"}, - { - "role": "user", - "content": "What's the weather like in San Francisco, Tokyo, and Paris?", - }, - ] - tools = [ - { - "type": "function", - "function": litellm.utils.function_to_dict(get_current_weather), - } - ] - OPENAI_GPT4oMINI = { - "messages": messages, - "model": "gpt-4o-mini", - "temperature": 0.0, - "tools": tools, - "stream": True, - } - LLAMA3_3 = { - "messages": messages, - "model": "groq/openai/gpt-oss-120b", - "api_base": "https://api.groq.com/openai/v1", - "temperature": 0.0, - "tools": tools, - "stream": True, - } - return OPENAI_GPT4oMINI, LLAMA3_3 - - -def execute_completion(opts: dict): - partial_streaming_chunks = [] - response_gen = litellm.completion(**opts) - for i, part in enumerate(response_gen): - partial_streaming_chunks.append(part) - print("\n\n") - print(f"partial_streaming_chunks: {partial_streaming_chunks}") - print("\n\n") - assembly = litellm.stream_chunk_builder(partial_streaming_chunks) - print( - f"assembly.choices[0].message.tool_calls: {assembly.choices[0].message.tool_calls}" - ) - print(assembly.choices[0].message.tool_calls) - for tool_call in assembly.choices[0].message.tool_calls: - json.loads( - tool_call.function.arguments - ) # assert valid json - https://github.com/BerriAI/litellm/issues/10034 - - -def test_grok_bug(load_env): - litellm.set_verbose = True - _, LLAMA3_3 = load_env - execute_completion(LLAMA3_3) diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py deleted file mode 100644 index c6582754c3a..00000000000 --- a/tests/local_testing/test_streaming.py +++ /dev/null @@ -1,1990 +0,0 @@ -#### What this tests #### -# This tests streaming for the completion endpoint - -import asyncio -import json -import os -import time -import traceback -from typing import Final, Tuple - -import pytest -from dotenv import load_dotenv -from pydantic import BaseModel - -import litellm.litellm_core_utils -import litellm.litellm_core_utils.litellm_logging -from litellm._uuid import uuid -from litellm.types.utils import ModelResponseStream - -load_dotenv() - -import litellm -from litellm import ( - AuthenticationError, - ModelResponse, - RateLimitError, - acompletion, - completion, -) - -litellm.logging = False -litellm.set_verbose = True -litellm.num_retries = 3 -litellm.cache = None - -score = 0 - - -def logger_fn(model_call_object: dict): - print(f"model call details: {model_call_object}") - - -user_message = "Hello, how are you?" -messages = [{"content": user_message, "role": "user"}] - - -first_openai_chunk_example = { - "id": "chatcmpl-7zSKLBVXnX9dwgRuDYVqVVDsgh2yp", - "object": "chat.completion.chunk", - "created": 1694881253, - "model": "gpt-4-0613", - "choices": [ - { - "index": 0, - "delta": {"role": "assistant", "content": ""}, - "finish_reason": None, # it's null - } - ], -} - - -def validate_first_format(chunk): - # write a test to make sure chunk follows the same format as first_openai_chunk_example - assert isinstance(chunk, ModelResponseStream), "Chunk should be a dictionary." - assert isinstance(chunk["id"], str), "'id' should be a string." - assert isinstance(chunk["object"], str), "'object' should be a string." - assert isinstance(chunk["created"], int), "'created' should be an integer." - assert isinstance(chunk["model"], str), "'model' should be a string." - assert isinstance(chunk["choices"], list), "'choices' should be a list." - assert getattr(chunk, "usage", None) is None, "Chunk cannot contain usage" - - for choice in chunk["choices"]: - assert isinstance(choice["index"], int), "'index' should be an integer." - assert isinstance( - choice["delta"]["role"], str - ), f"'role' should be a string. Got {choice['delta']['role']}" - assert "messages" not in choice - # openai v1.0.0 returns content as None - assert (choice["finish_reason"] is None) or isinstance( - choice["finish_reason"], str - ), "'finish_reason' should be None or a string." - - -second_openai_chunk_example = { - "id": "chatcmpl-7zSKLBVXnX9dwgRuDYVqVVDsgh2yp", - "object": "chat.completion.chunk", - "created": 1694881253, - "model": "gpt-4-0613", - "choices": [ - {"index": 0, "delta": {"content": "Hello"}, "finish_reason": None} # it's null - ], -} - - -def validate_second_format(chunk): - assert isinstance(chunk, ModelResponseStream), "Chunk should be a dictionary." - assert isinstance(chunk["id"], str), "'id' should be a string." - assert isinstance(chunk["object"], str), "'object' should be a string." - assert isinstance(chunk["created"], int), "'created' should be an integer." - assert isinstance(chunk["model"], str), "'model' should be a string." - assert isinstance(chunk["choices"], list), "'choices' should be a list." - assert getattr(chunk, "usage", None) is None, "Chunk cannot contain usage" - - for choice in chunk["choices"]: - assert isinstance(choice["index"], int), "'index' should be an integer." - assert hasattr(choice["delta"], "role"), "'role' should be a string." - # openai v1.0.0 returns content as None - assert (choice["finish_reason"] is None) or isinstance( - choice["finish_reason"], str - ), "'finish_reason' should be None or a string." - - -last_openai_chunk_example = { - "id": "chatcmpl-7zSKLBVXnX9dwgRuDYVqVVDsgh2yp", - "object": "chat.completion.chunk", - "created": 1694881253, - "model": "gpt-4-0613", - "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], -} - -""" -Final chunk (sdk): -chunk: ChatCompletionChunk(id='chatcmpl-96mM3oNBlxh2FDWVLKsgaFBBcULmI', -choices=[Choice(delta=ChoiceDelta(content=None, function_call=None, role=None, -tool_calls=None), finish_reason='stop', index=0, logprobs=None)], -created=1711402871, model='gpt-3.5-turbo-0125', object='chat.completion.chunk', system_fingerprint='fp_3bc1b5746c') -""" - - -def validate_last_format(chunk): - """ - Ensure last chunk has no remaining content or tools - """ - assert isinstance(chunk, ModelResponseStream), "Chunk should be a dictionary." - assert isinstance(chunk["id"], str), "'id' should be a string." - assert isinstance(chunk["object"], str), "'object' should be a string." - assert isinstance(chunk["created"], int), "'created' should be an integer." - assert isinstance(chunk["model"], str), "'model' should be a string." - assert isinstance(chunk["choices"], list), "'choices' should be a list." - assert getattr(chunk, "usage", None) is None, "Chunk cannot contain usage" - - for choice in chunk["choices"]: - assert isinstance(choice["index"], int), "'index' should be an integer." - assert choice["delta"]["content"] is None - assert choice["delta"]["function_call"] is None - assert choice["delta"]["role"] is None - assert choice["delta"]["tool_calls"] is None - assert isinstance( - choice["finish_reason"], str - ), "'finish_reason' should be a string." - - -def streaming_format_tests(idx, chunk) -> Tuple[str, bool]: - extracted_chunk = "" - finished = False - print(f"chunk: {chunk}") - if idx == 0: # ensure role assistant is set - validate_first_format(chunk=chunk) - role = chunk["choices"][0]["delta"]["role"] - assert role == "assistant" - elif idx == 1: # second chunk - validate_second_format(chunk=chunk) - if idx != 0: # ensure no role - if "role" in chunk["choices"][0]["delta"]: - pass # openai v1.0.0+ passes role = None - if chunk["choices"][0][ - "finish_reason" - ]: # ensure finish reason is only in last chunk - validate_last_format(chunk=chunk) - finished = True - if ( - "content" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["content"] is not None - ): - extracted_chunk = chunk["choices"][0]["delta"]["content"] - print(f"extracted chunk: {extracted_chunk}") - return extracted_chunk, finished - - -tools_schema = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - }, - } -] - - -def test_completion_azure_stream_special_char(): - litellm.set_verbose = True - messages = [{"role": "user", "content": "hi. respond with the tag only"}] - response = completion(model="azure/gpt-4.1-mini", messages=messages, stream=True) - response_str = "" - for part in response: - response_str += part.choices[0].delta.content or "" - print(f"response_str: {response_str}") - assert len(response_str) > 0 - - - - -@pytest.mark.flaky(retries=5, delay=1) -def test_completion_azure_stream(): - try: - litellm.set_verbose = False - messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - { - "role": "user", - "content": "how does a court case get to the Supreme Court?", - }, - ] - response = completion( - model="azure/gpt-4.1-mini", messages=messages, stream=True, max_tokens=50 - ) - complete_response = "" - # Add any assertions here to check the response - for idx, init_chunk in enumerate(response): - chunk, finished = streaming_format_tests(idx, init_chunk) - complete_response += chunk - custom_llm_provider = init_chunk._hidden_params["custom_llm_provider"] - print(f"custom_llm_provider: {custom_llm_provider}") - assert custom_llm_provider == "azure" - if finished: - assert isinstance(init_chunk.choices[0], litellm.utils.StreamingChoices) - break - if complete_response.strip() == "": - raise Exception("Empty response received") - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - - - -@pytest.mark.parametrize( - "model", - [ - # "claude-3-5-haiku-20241022", - # "mistral/mistral-small-latest", - "openrouter/openai/gpt-4o-mini", - ], -) -def test_completion_model_stream(model): - try: - messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - { - "role": "user", - "content": "how does a court case get to the Supreme Court?", - }, - ] - response = completion( - model=model, messages=messages, stream=True, max_tokens=50 - ) - complete_response = "" - # Add any assertions here to check the response - for idx, chunk in enumerate(response): - chunk, finished = streaming_format_tests(idx, chunk) - if finished: - break - complete_response += chunk - if complete_response.strip() == "": - raise Exception("Empty response received") - print(f"completion_response: {complete_response}") - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -@pytest.mark.parametrize( - "sync_mode", - [True, False], -) # , -@pytest.mark.asyncio -@pytest.mark.flaky(retries=3, delay=1) -async def test_completion_gemini_stream(sync_mode): - try: - litellm.turn_on_debug() - print("Streaming gemini response") - function1 = [ - { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - } - ] - messages = [ - { - "role": "user", - "content": "What is the weather like in Boston, MA?. You must provide me with a tool call in your response.", - } - ] - print("testing gemini streaming") - complete_response = "" - # Add any assertions here to check the response - non_empty_chunks = 0 - chunks = [] - if sync_mode: - response = completion( - model="gemini/gemini-2.5-flash-lite", - messages=messages, - stream=True, - functions=function1, - ) - - for idx, chunk in enumerate(response): - print(chunk) - chunks.append(chunk) - # print(chunk.choices[0].delta) - chunk, finished = streaming_format_tests(idx, chunk) - if finished: - break - non_empty_chunks += 1 - complete_response += chunk - else: - response = await litellm.acompletion( - model="gemini/gemini-2.5-flash-lite", - messages=messages, - stream=True, - functions=function1, - ) - - idx = 0 - async for chunk in response: - print(chunk) - chunks.append(chunk) - # print(chunk.choices[0].delta) - chunk, finished = streaming_format_tests(idx, chunk) - if finished: - break - non_empty_chunks += 1 - complete_response += chunk - idx += 1 - - # if complete_response.strip() == "": - # raise Exception("Empty response received") - print(f"completion_response: {complete_response}") - - complete_response = litellm.stream_chunk_builder( - chunks=chunks, messages=messages - ) - - assert complete_response.choices[0].message.function_call is not None - - # assert non_empty_chunks > 1 - except litellm.InternalServerError as e: - pass - except litellm.RateLimitError as e: - pass - except Exception as e: - # if "429 Resource has been exhausted": - # return - pytest.fail(f"Error occurred: {e}") - - - - - - -@pytest.mark.flaky(retries=3, delay=1) -def test_completion_mistral_api_mistral_large_function_call_with_streaming(): - litellm.set_verbose = True - tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - }, - } - ] - messages = [ - { - "role": "user", - "content": "What's the weather like in Boston today in fahrenheit?", - } - ] - try: - # test without max tokens - response = completion( - model="mistral/mistral-medium-latest", - messages=messages, - tools=tools, - tool_choice="auto", - stream=True, - ) - saw_function_call_chunk = False - for chunk in response: - print(f"chunk in response: {chunk}") - assert chunk._hidden_params["custom_llm_provider"] == "mistral" - if len(chunk.choices) == 0: - continue - if chunk.choices[0].finish_reason is not None: # last chunk - validate_final_streaming_function_calling_chunk(chunk=chunk) - break - tool_calls = chunk.choices[0].delta.tool_calls - if tool_calls is None: - continue - assert tool_calls[0].function.arguments is not None - assert isinstance(tool_calls[0].function.arguments, str) - if not saw_function_call_chunk: - if chunk.choices[0].delta.role is not None: - validate_first_streaming_function_calling_chunk(chunk=chunk) - else: - validate_second_streaming_function_calling_chunk(chunk=chunk) - saw_function_call_chunk = True - else: - validate_second_streaming_function_calling_chunk(chunk=chunk) - assert saw_function_call_chunk - except litellm.RateLimitError: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - - - -def test_completion_claude_stream_bad_key(): - try: - litellm.cache = None - litellm.set_verbose = True - api_key = "bad-key" - messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - { - "role": "user", - "content": "how does a court case get to the Supreme Court?", - }, - ] - response = completion( - model="claude-haiku-4-5-20251001", - messages=messages, - stream=True, - max_tokens=50, - api_key=api_key, - ) - complete_response = "" - # Add any assertions here to check the response - has_finish_reason = False - for idx, chunk in enumerate(response): - chunk, finished = streaming_format_tests(idx, chunk) - if finished: - has_finish_reason = True - break - complete_response += chunk - if has_finish_reason == False: - raise Exception("finish reason not set") - if complete_response.strip() == "": - raise Exception("Empty response received") - print(f"1234completion_response: {complete_response}") - raise Exception("Auth error not raised") - except AuthenticationError as e: - print("Auth Error raised") - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -@pytest.mark.parametrize("provider", ["vertex_ai_beta"]) # "" -def test_vertex_ai_stream(provider): - from test_amazing_vertex_completion import ( - load_vertex_ai_credentials, - ) - - load_vertex_ai_credentials() - litellm.set_verbose = True - - test_models = ["gemini-2.5-flash-lite"] - for model in test_models: - try: - print("making request", model) - response = completion( - model="{}/{}".format(provider, model), - messages=[ - {"role": "user", "content": "Hey, how's it going?"}, - { - "role": "assistant", - "content": "I'm doing well. Would like to hear the rest of the story?", - }, - {"role": "user", "content": "Na"}, - { - "role": "assistant", - "content": "No problem, is there anything else i can help you with today?", - }, - { - "role": "user", - "content": "I think you're getting cut off sometimes", - }, - ], - stream=True, - ) - complete_response = "" - is_finished = False - for idx, chunk in enumerate(response): - print(f"chunk in response: {chunk}") - chunk, finished = streaming_format_tests(idx, chunk) - if finished: - is_finished = True - break - complete_response += chunk - if complete_response.strip() == "": - raise Exception("Empty response received") - print(f"completion_response: {complete_response}") - assert is_finished == True - - except litellm.RateLimitError as e: - pass - except litellm.exceptions.MidStreamFallbackError as e: - # Streaming 429s are wrapped in MidStreamFallbackError so the - # Router can fall back; treat as a transient rate-limit pass. - if not isinstance(e.original_exception, litellm.RateLimitError): - pytest.fail(f"Error occurred: {e}") - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - - - -@pytest.mark.parametrize("sync_mode", [True, False]) # -@pytest.mark.parametrize( - "model, region", - [ - # ["bedrock/ai21.jamba-instruct-v1:0", "us-east-1"], - ["us.anthropic.claude-sonnet-4-5-20250929-v1:0", None], - # ["mistral.mistral-7b-instruct-v0:2", None], - # ["meta.llama3-8b-instruct-v1:0", None], - ], -) -@pytest.mark.asyncio -@pytest.mark.flaky(retries=3, delay=1) -async def test_bedrock_httpx_streaming(sync_mode, model, region): - try: - litellm.set_verbose = True - if sync_mode: - final_chunk: Optional[litellm.ModelResponse] = None - response: litellm.CustomStreamWrapper = completion( # type: ignore - model=model, - messages=messages, - max_tokens=10, # type: ignore - stream=True, - aws_region_name=region, - ) - complete_response = "" - # Add any assertions here to check the response - has_finish_reason = False - for idx, chunk in enumerate(response): - final_chunk = chunk - chunk, finished = streaming_format_tests(idx, chunk) - if finished: - has_finish_reason = True - break - complete_response += chunk - if has_finish_reason is False: - raise Exception("finish reason not set") - if complete_response.strip() == "": - raise Exception("Empty response received") - else: - response: litellm.CustomStreamWrapper = await litellm.acompletion( # type: ignore - model=model, - messages=messages, - max_tokens=100, # type: ignore - stream=True, - aws_region_name=region, - ) - complete_response = "" - # Add any assertions here to check the response - has_finish_reason = False - idx = 0 - final_chunk: Optional[litellm.ModelResponse] = None - async for chunk in response: - final_chunk = chunk - chunk, finished = streaming_format_tests(idx, chunk) - if finished: - has_finish_reason = True - break - complete_response += chunk - idx += 1 - if has_finish_reason == False: - raise Exception("finish reason not set") - if complete_response.strip() == "": - raise Exception("Empty response received") - print(f"completion_response: {complete_response}\n\nFinalChunk: {final_chunk}") - except RateLimitError as e: - print("got rate limit error=", e) - pass - except litellm.Timeout: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -def test_bedrock_claude_3_streaming(): - try: - litellm.set_verbose = True - response: ModelResponse = completion( # type: ignore - model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", - messages=messages, - max_tokens=10, # type: ignore - stream=True, - ) - complete_response = "" - # Add any assertions here to check the response - has_finish_reason = False - for idx, chunk in enumerate(response): - chunk, finished = streaming_format_tests(idx, chunk) - if finished: - has_finish_reason = True - break - complete_response += chunk - if has_finish_reason == False: - raise Exception("finish reason not set") - if complete_response.strip() == "": - raise Exception("Empty response received") - print(f"completion_response: {complete_response}") - except RateLimitError: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.parametrize( - "model", - [ - "claude-haiku-4-5-20251001", - "bedrock/mistral.mistral-7b-instruct-v0:2", - "gpt-3.5-turbo", - ], -) -@pytest.mark.asyncio -async def test_parallel_streaming_requests(sync_mode, model): - """ - Important prod test. - """ - try: - import threading - - litellm.set_verbose = True - messages = [ - {"role": "system", "content": "Be helpful"}, - {"role": "user", "content": "What do you know?"}, - ] - - def sync_test_streaming(): - response: litellm.CustomStreamWrapper = litellm.completion( # type: ignore - model=model, - messages=messages, - stream=True, - max_tokens=10, - timeout=10, - ) - complete_response = "" - # Add any assertions here to-check the response - num_finish_reason = 0 - for chunk in response: - print(f"chunk: {chunk}") - if isinstance(chunk, ModelResponseStream): - if chunk.choices[0].finish_reason is not None: - num_finish_reason += 1 - assert num_finish_reason == 1 - - async def test_streaming(): - response: litellm.CustomStreamWrapper = await litellm.acompletion( # type: ignore - model=model, - messages=messages, - stream=True, - max_tokens=10, - timeout=10, - ) - complete_response = "" - # Add any assertions here to-check the response - num_finish_reason = 0 - async for chunk in response: - print(f"type of chunk: {type(chunk)}") - if isinstance(chunk, ModelResponseStream): - print(f"OUTSIDE CHUNK: {chunk.choices[0]}") - if chunk.choices[0].finish_reason is not None: - num_finish_reason += 1 - assert num_finish_reason == 1 - - tasks = [] - for _ in range(2): - if sync_mode == False: - tasks.append(test_streaming()) - else: - thread = threading.Thread(target=sync_test_streaming) - thread.start() - tasks.append(thread) - - if sync_mode == False: - await asyncio.gather(*tasks) - else: - # Wait for all threads to complete - for thread in tasks: - thread.join() - - except RateLimitError: - pass - except litellm.Timeout: - pass - except litellm.ServiceUnavailableError as e: - if model == "predibase/llama-3-8b-instruct": - pass - else: - pytest.fail(f"Service Unavailable Error got{str(e)}") - except litellm.InternalServerError as e: - if "predibase" in str(e).lower(): - # only skip internal server error from predibase - their endpoint seems quite unstable - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - - - - - -def test_completion_bedrock_mistral_stream(): - try: - litellm.set_verbose = False - response = completion( - model="bedrock/mistral.mixtral-8x7b-instruct-v0:1", - messages=[ - { - "role": "user", - "content": "Be as verbose as possible and give as many details as possible, how does a court case get to the Supreme Court?", - } - ], - temperature=1, - max_tokens=20, - stream=True, - ) - print(response) - complete_response = "" - has_finish_reason = False - # Add any assertions here to check the response - for idx, chunk in enumerate(response): - # print - chunk, finished = streaming_format_tests(idx, chunk) - has_finish_reason = finished - complete_response += chunk - if finished: - break - if has_finish_reason is False: - raise Exception("finish reason not set for last chunk") - if complete_response.strip() == "": - raise Exception("Empty response received") - print(f"completion_response: {complete_response}") - except RateLimitError: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - - - - - - - -# test on openai completion call -def test_openai_chat_completion_call(): - litellm.set_verbose = False - litellm.return_response_headers = True - response = completion(model="gpt-3.5-turbo", messages=messages, stream=True) - assert isinstance( - response._hidden_params["additional_headers"][ - "llm_provider-x-ratelimit-remaining-requests" - ], - str, - ) - - print(f"response._hidden_params: {response._hidden_params}") - complete_response = "" - start_time = time.time() - for idx, chunk in enumerate(response): - chunk, finished = streaming_format_tests(idx, chunk) - print(f"outside chunk: {chunk}") - if finished: - break - complete_response += chunk - # print(f'complete_chunk: {complete_response}') - if complete_response.strip() == "": - raise Exception("Empty response received") - print(f"complete response: {complete_response}") - - -@pytest.mark.parametrize( - "model", - [ - "gpt-3.5-turbo", - "claude-haiku-4-5-20251001", - "o1", - ], -) -@pytest.mark.parametrize( - "sync", - [True, False], -) -@pytest.mark.asyncio -@pytest.mark.flaky(retries=6, delay=10) -async def test_openai_stream_options_call(model, sync): - litellm.enable_preview_features = True - litellm.set_verbose = True - usage = None - chunks = [] - if sync: - response = litellm.completion( - model=model, - messages=[ - {"role": "user", "content": "say GM - we're going to make it "}, - ], - stream=True, - stream_options={"include_usage": True}, - ) - for chunk in response: - print("chunk: ", chunk) - chunks.append(chunk) - else: - response = await litellm.acompletion( - model=model, - messages=[{"role": "user", "content": "say GM - we're going to make it "}], - stream=True, - stream_options={"include_usage": True}, - ) - - async for chunk in response: - print("chunk: ", chunk) - chunks.append(chunk) - - last_chunk = chunks[-1] - print("last chunk: ", last_chunk) - - """ - Assert that: - - Last Chunk includes Usage - - All chunks prior to last chunk have usage=None - """ - - assert last_chunk.usage is not None - assert isinstance(last_chunk.usage, litellm.Usage) - assert last_chunk.usage.total_tokens > 0 - assert last_chunk.usage.prompt_tokens > 0 - assert last_chunk.usage.completion_tokens > 0 - - # assert all non last chunks have usage=None - # Improved assertion with detailed error message - non_last_chunks_with_usage = [ - chunk - for chunk in chunks[:-1] - if hasattr(chunk, "usage") and chunk.usage is not None - ] - assert ( - not non_last_chunks_with_usage - ), f"Non-last chunks with usage not None:\n" + "\n".join( - f"Chunk ID: {chunk.id}, Usage: {chunk.usage}, Content: {chunk.choices[0].delta.content}" - for chunk in non_last_chunks_with_usage - ) - - -def test_openai_stream_options_call_text_completion() -> None: - chunks: Final = tuple( - litellm.text_completion( - model="gpt-6-luna", - reasoning_effort="none", - prompt="say GM - we're going to make it ", - stream=True, - stream_options={"include_usage": True}, - max_tokens=10, - ) - ) - assert chunks - assert chunks[-1].usage is not None - assert chunks[-1].usage.total_tokens > 0 - assert chunks[-1].usage.prompt_tokens > 0 - assert chunks[-1].usage.completion_tokens > 0 - assert all(chunk.usage is None for chunk in chunks[:-1]) - assert any(chunk.choices[0].text for chunk in chunks) - - - - -#### Test Function calling + streaming #### - - -def test_completion_openai_with_functions() -> None: - functions: Final = [ - { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - } - ] - messages: Final = [{"role": "user", "content": "what's the weather in SF"}] - chunks: Final = tuple( - completion( - model="gpt-6-luna", - reasoning_effort="none", - messages=messages, - functions=functions, - function_call={"name": "get_current_weather"}, - stream=True, - max_tokens=128, - ) - ) - response: Final = litellm.stream_chunk_builder(chunks, messages=messages) - assert response is not None - function_call: Final = response.choices[0].message.function_call - assert function_call is not None - assert function_call.name == "get_current_weather" - assert json.loads(function_call.arguments)["location"] - assert sum(chunk.choices[0].finish_reason is not None for chunk in chunks) == 1 - - -#### Test Async streaming #### - - -async def completion_call(): - try: - response = completion( - model="gpt-3.5-turbo", - messages=messages, - stream=True, - logger_fn=logger_fn, - max_tokens=10, - ) - print(f"response: {response}") - complete_response = "" - start_time = time.time() - # Change for loop to async for loop - idx = 0 - async for chunk in response: - chunk, finished = streaming_format_tests(idx, chunk) - if finished: - break - complete_response += chunk - idx += 1 - if complete_response.strip() == "": - raise Exception("Empty response received") - print(f"complete response: {complete_response}") - except Exception: - print(f"error occurred: {traceback.format_exc()}") - pass - - -#### Test Function Calling + Streaming #### - -final_openai_function_call_example = { - "id": "chatcmpl-7zVNA4sXUftpIg6W8WlntCyeBj2JY", - "object": "chat.completion", - "created": 1694892960, - "model": "gpt-3.5-turbo", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": None, - "function_call": { - "name": "get_current_weather", - "arguments": '{\n "location": "Boston, MA"\n}', - }, - }, - "finish_reason": "function_call", - } - ], - "usage": {"prompt_tokens": 82, "completion_tokens": 18, "total_tokens": 100}, -} - -function_calling_output_structure = { - "id": str, - "object": str, - "created": int, - "model": str, - "choices": [ - { - "index": int, - "message": { - "role": str, - "content": (type(None), str), - "function_call": {"name": str, "arguments": str}, - }, - "finish_reason": str, - } - ], - "usage": {"prompt_tokens": int, "completion_tokens": int, "total_tokens": int}, -} - - -def validate_final_structure(item, structure=function_calling_output_structure): - if isinstance(item, list): - if not all(validate_final_structure(i, structure[0]) for i in item): - return Exception( - "Function calling final output doesn't match expected output format" - ) - elif isinstance(item, dict): - if not all( - k in item and validate_final_structure(item[k], v) - for k, v in structure.items() - ): - return Exception( - "Function calling final output doesn't match expected output format" - ) - else: - if not isinstance(item, structure): - return Exception( - "Function calling final output doesn't match expected output format" - ) - return True - - -first_openai_function_call_example = { - "id": "chatcmpl-7zVRoE5HjHYsCMaVSNgOjzdhbS3P0", - "object": "chat.completion.chunk", - "created": 1694893248, - "model": "gpt-3.5-turbo", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "content": None, - "function_call": {"name": "get_current_weather", "arguments": ""}, - }, - "finish_reason": None, - } - ], -} - - -def validate_first_function_call_chunk_structure(item): - if not (isinstance(item, dict) or isinstance(item, litellm.ModelResponse)): - raise Exception(f"Incorrect format, type of item: {type(item)}") - - required_keys = {"id", "object", "created", "model", "choices"} - for key in required_keys: - if key not in item: - raise Exception("Incorrect format") - - if not isinstance(item["choices"], list) or not item["choices"]: - raise Exception("Incorrect format") - - required_keys_in_choices_array = {"index", "delta", "finish_reason"} - for choice in item["choices"]: - if not ( - isinstance(choice, dict) - or isinstance(choice, litellm.utils.StreamingChoices) - ): - raise Exception(f"Incorrect format, type of choice: {type(choice)}") - for key in required_keys_in_choices_array: - if key not in choice: - raise Exception("Incorrect format") - - if not ( - isinstance(choice["delta"], dict) - or isinstance(choice["delta"], litellm.utils.Delta) - ): - raise Exception( - f"Incorrect format, type of choice: {type(choice['delta'])}" - ) - - required_keys_in_delta = {"role", "content", "function_call"} - for key in required_keys_in_delta: - if key not in choice["delta"]: - raise Exception("Incorrect format") - - if not ( - isinstance(choice["delta"]["function_call"], dict) - or isinstance(choice["delta"]["function_call"], BaseModel) - ): - raise Exception( - f"Incorrect format, type of function call: {type(choice['delta']['function_call'])}" - ) - - required_keys_in_function_call = {"name", "arguments"} - for key in required_keys_in_function_call: - if not hasattr(choice["delta"]["function_call"], key): - raise Exception( - f"Incorrect format, expected key={key}; actual keys: {choice['delta']['function_call']}, eval: {hasattr(choice['delta']['function_call'], key)}" - ) - - return True - - -second_function_call_chunk_format = { - "id": "chatcmpl-7zVRoE5HjHYsCMaVSNgOjzdhbS3P0", - "object": "chat.completion.chunk", - "created": 1694893248, - "model": "gpt-3.5-turbo", - "choices": [ - { - "index": 0, - "delta": {"function_call": {"arguments": "{\n"}}, - "finish_reason": None, - } - ], -} - - -def validate_second_function_call_chunk_structure(data): - if not isinstance(data, dict): - raise Exception("Incorrect format") - - required_keys = {"id", "object", "created", "model", "choices"} - for key in required_keys: - if key not in data: - raise Exception("Incorrect format") - - if not isinstance(data["choices"], list) or not data["choices"]: - raise Exception("Incorrect format") - - required_keys_in_choices_array = {"index", "delta", "finish_reason"} - for choice in data["choices"]: - if not isinstance(choice, dict): - raise Exception("Incorrect format") - for key in required_keys_in_choices_array: - if key not in choice: - raise Exception("Incorrect format") - - if ( - "function_call" not in choice["delta"] - or "arguments" not in choice["delta"]["function_call"] - ): - raise Exception("Incorrect format") - - return True - - -final_function_call_chunk_example = { - "id": "chatcmpl-7zVRoE5HjHYsCMaVSNgOjzdhbS3P0", - "object": "chat.completion.chunk", - "created": 1694893248, - "model": "gpt-3.5-turbo", - "choices": [{"index": 0, "delta": {}, "finish_reason": "function_call"}], -} - - -def validate_final_function_call_chunk_structure(data): - if not (isinstance(data, dict) or isinstance(data, litellm.ModelResponse)): - raise Exception("Incorrect format") - - required_keys = {"id", "object", "created", "model", "choices"} - for key in required_keys: - if key not in data: - raise Exception("Incorrect format") - - if not isinstance(data["choices"], list) or not data["choices"]: - raise Exception("Incorrect format") - - required_keys_in_choices_array = {"index", "delta", "finish_reason"} - for choice in data["choices"]: - if not ( - isinstance(choice, dict) or isinstance(choice["delta"], litellm.utils.Delta) - ): - raise Exception("Incorrect format") - for key in required_keys_in_choices_array: - if key not in choice: - raise Exception("Incorrect format") - - return True - - -def streaming_and_function_calling_format_tests(idx, chunk): - extracted_chunk = "" - finished = False - print(f"idx: {idx}") - print(f"chunk: {chunk}") - decision = False - if idx == 0: # ensure role assistant is set - decision = validate_first_function_call_chunk_structure(chunk) - role = chunk["choices"][0]["delta"]["role"] - assert role == "assistant" - elif idx != 0: # second chunk - try: - decision = validate_second_function_call_chunk_structure(data=chunk) - except Exception: # check if it's the last chunk (returns an empty delta {} ) - decision = validate_final_function_call_chunk_structure(data=chunk) - finished = True - if "content" in chunk["choices"][0]["delta"]: - extracted_chunk = chunk["choices"][0]["delta"]["content"] - if decision == False: - raise Exception("incorrect format") - return extracted_chunk, finished - - -@pytest.mark.parametrize( - "model", - [ - # "gpt-3.5-turbo", - # "anthropic.claude-3-sonnet-20240229-v1:0", - "claude-haiku-4-5-20251001", - ], -) -def test_streaming_and_function_calling(model): - import json - - tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - }, - } - ] - - messages = [{"role": "user", "content": "What is the weather like in Boston?"}] - try: - # litellm.set_verbose = True - response: litellm.CustomStreamWrapper = completion( - model=model, - tools=tools, - messages=messages, - stream=True, - tool_choice="required", - ) # type: ignore - # Add any assertions here to check the response - json_str = "" - for idx, chunk in enumerate(response): - # continue - # print("\n{}\n".format(chunk)) - if idx == 0: - assert ( - chunk.choices[0].delta.tool_calls[0].function.arguments is not None - ) - assert isinstance( - chunk.choices[0].delta.tool_calls[0].function.arguments, str - ) - if chunk.choices[0].delta.tool_calls is not None: - json_str += chunk.choices[0].delta.tool_calls[0].function.arguments - - print(json.loads(json_str)) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - raise e - - -def test_success_callback_streaming(): - def success_callback(kwargs, completion_response, start_time, end_time): - print( - { - "success": True, - "input": kwargs, - "output": completion_response, - "start_time": start_time, - "end_time": end_time, - } - ) - - litellm.success_callback = [success_callback] - - messages = [{"role": "user", "content": "hello"}] - print("TESTING LITELLM COMPLETION CALL") - response = litellm.completion( - model="gpt-3.5-turbo", - messages=messages, - stream=True, - max_tokens=5, - ) - print(response) - - for chunk in response: - print(chunk["choices"][0]) - - -from typing import List, Optional - -#### STREAMING + FUNCTION CALLING ### - - -class Function(BaseModel): - name: str - arguments: str - - -class ToolCalls(BaseModel): - index: int - id: str - type: str - function: Function - - -class Delta(BaseModel): - role: str - content: Optional[str] - tool_calls: List[ToolCalls] - - -class Choices(BaseModel): - index: int - delta: Delta - logprobs: Optional[str] - finish_reason: Optional[str] - - -class Chunk(BaseModel): - id: str - object: str - created: int - model: str - # system_fingerprint: str - choices: List[Choices] - - -def validate_first_streaming_function_calling_chunk(chunk: ModelResponse): - chunk_instance = Chunk(**chunk.model_dump()) - - -### Chunk 1 - - -# { -# "id": "chatcmpl-8vdVjtzxc0JqGjq93NxC79dMp6Qcs", -# "object": "chat.completion.chunk", -# "created": 1708747267, -# "model": "gpt-3.5-turbo-0125", -# "system_fingerprint": "fp_86156a94a0", -# "choices": [ -# { -# "index": 0, -# "delta": { -# "role": "assistant", -# "content": null, -# "tool_calls": [ -# { -# "index": 0, -# "id": "call_oN10vaaC9iA8GLFRIFwjCsN7", -# "type": "function", -# "function": { -# "name": "get_current_weather", -# "arguments": "" -# } -# } -# ] -# }, -# "logprobs": null, -# "finish_reason": null -# } -# ] -# } -class Function2(BaseModel): - arguments: str - - -class ToolCalls2(BaseModel): - index: int - function: Optional[Function2] - - -class Delta2(BaseModel): - tool_calls: List[ToolCalls2] - - -class Choices2(BaseModel): - index: int - delta: Delta2 - logprobs: Optional[str] - finish_reason: Optional[str] - - -class Chunk2(BaseModel): - id: str - object: str - created: int - model: str - system_fingerprint: Optional[str] - choices: List[Choices2] - - -## Chunk 2 - -# { -# "id": "chatcmpl-8vdVjtzxc0JqGjq93NxC79dMp6Qcs", -# "object": "chat.completion.chunk", -# "created": 1708747267, -# "model": "gpt-3.5-turbo-0125", -# "system_fingerprint": "fp_86156a94a0", -# "choices": [ -# { -# "index": 0, -# "delta": { -# "tool_calls": [ -# { -# "index": 0, -# "function": { -# "arguments": "{\"" -# } -# } -# ] -# }, -# "logprobs": null, -# "finish_reason": null -# } -# ] -# } - - -def validate_second_streaming_function_calling_chunk(chunk: ModelResponse): - chunk_instance = Chunk2(**chunk.model_dump()) - - -class Delta3(BaseModel): - content: Optional[str] = None - role: Optional[str] = None - function_call: Optional[dict] = None - tool_calls: Optional[List] = None - - -class Choices3(BaseModel): - index: int - delta: Delta3 - logprobs: Optional[str] - finish_reason: str - - -class Chunk3(BaseModel): - id: str - object: str - created: int - model: str - # system_fingerprint: str - choices: List[Choices3] - - -def validate_final_streaming_function_calling_chunk(chunk: ModelResponse): - chunk_instance = Chunk3(**chunk.model_dump()) - - -def test_azure_streaming_and_function_calling(): - tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - }, - } - ] - messages = [{"role": "user", "content": "What is the weather like in Boston?"}] - try: - response = completion( - model="azure/gpt-4.1-mini", - tools=tools, - tool_choice="auto", - messages=messages, - stream=True, - api_base=os.getenv("AZURE_AI_API_BASE"), - api_key=os.getenv("AZURE_AI_API_KEY"), - api_version="2024-02-15-preview", - ) - # Add any assertions here to check the response - for idx, chunk in enumerate(response): - print(f"chunk: {chunk}") - if idx == 0: - assert ( - chunk.choices[0].delta.tool_calls[0].function.arguments is not None - ) - assert isinstance( - chunk.choices[0].delta.tool_calls[0].function.arguments, str - ) - validate_first_streaming_function_calling_chunk(chunk=chunk) - elif idx == 1: - validate_second_streaming_function_calling_chunk(chunk=chunk) - elif chunk.choices[0].finish_reason is not None: # last chunk - validate_final_streaming_function_calling_chunk(chunk=chunk) - - except Exception as e: - pytest.fail(f"Error occurred: {e}") - raise e - - -@pytest.mark.asyncio -async def test_azure_astreaming_and_function_calling(): - - tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - }, - } - ] - messages = [ - { - "role": "user", - "content": f"What is the weather like in Boston? {uuid.uuid4()}", - } - ] - from litellm.caching.caching import Cache - - litellm.cache = Cache( - type="redis", - host=os.environ["REDIS_HOST"], - port=os.environ["REDIS_PORT"], - password=os.environ["REDIS_PASSWORD"], - ) - try: - litellm.set_verbose = True - response = await litellm.acompletion( - model="azure/gpt-4.1-mini", - tools=tools, - tool_choice="auto", - messages=messages, - stream=True, - api_base=os.getenv("AZURE_AI_API_BASE"), - api_key=os.getenv("AZURE_AI_API_KEY"), - api_version="2024-02-15-preview", - caching=True, - ) - # Add any assertions here to check the response - idx = 0 - async for chunk in response: - print(f"chunk: {chunk}") - if idx == 0: - assert ( - chunk.choices[0].delta.tool_calls[0].function.arguments is not None - ) - assert isinstance( - chunk.choices[0].delta.tool_calls[0].function.arguments, str - ) - validate_first_streaming_function_calling_chunk(chunk=chunk) - elif idx == 1: - validate_second_streaming_function_calling_chunk(chunk=chunk) - elif chunk.choices[0].finish_reason is not None: # last chunk - validate_final_streaming_function_calling_chunk(chunk=chunk) - idx += 1 - - ## CACHING TEST - print("\n\nCACHING TESTS\n\n") - response = await litellm.acompletion( - model="azure/gpt-4.1-mini", - tools=tools, - tool_choice="auto", - messages=messages, - stream=True, - api_base=os.getenv("AZURE_AI_API_BASE"), - api_key=os.getenv("AZURE_AI_API_KEY"), - api_version="2024-02-15-preview", - caching=True, - ) - # Add any assertions here to check the response - idx = 0 - async for chunk in response: - print(f"chunk: {chunk}") - if idx == 0: - assert ( - chunk.choices[0].delta.tool_calls[0].function.arguments is not None - ) - assert isinstance( - chunk.choices[0].delta.tool_calls[0].function.arguments, str - ) - validate_first_streaming_function_calling_chunk(chunk=chunk) - elif idx == 1 and chunk.choices[0].finish_reason is None: - validate_second_streaming_function_calling_chunk(chunk=chunk) - elif chunk.choices[0].finish_reason is not None: # last chunk - validate_final_streaming_function_calling_chunk(chunk=chunk) - idx += 1 - except Exception as e: - pytest.fail(f"Error occurred: {e}") - raise e - - -def test_completion_claude_3_function_call_with_streaming(): - tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - }, - } - ] - messages = [ - { - "role": "user", - "content": "What's the weather like in Boston today in fahrenheit?", - } - ] - try: - # test without max tokens - response = completion( - model="claude-haiku-4-5-20251001", - messages=messages, - tools=tools, - tool_choice="required", - stream=True, - ) - idx = 0 - for chunk in response: - print(f"chunk in response: {chunk}") - if idx == 0: - assert ( - chunk.choices[0].delta.tool_calls[0].function.arguments is not None - ) - assert isinstance( - chunk.choices[0].delta.tool_calls[0].function.arguments, str - ) - validate_first_streaming_function_calling_chunk(chunk=chunk) - elif idx == 1 and chunk.choices[0].finish_reason is None: - validate_second_streaming_function_calling_chunk(chunk=chunk) - elif chunk.choices[0].finish_reason is not None: # last chunk - assert "usage" in chunk._hidden_params - validate_final_streaming_function_calling_chunk(chunk=chunk) - idx += 1 - # raise Exception("it worked!") - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -@pytest.mark.parametrize( - "model", - [ - "gemini/gemini-2.5-flash-lite", - ], -) # -@pytest.mark.asyncio -async def test_acompletion_function_call_with_streaming(model): - litellm.set_verbose = True - tools = [ - { - "type": "function", - "function": { - "name": "generate_series_of_questions", - "description": "Generate a series of questions, given a topic.", - "parameters": { - "type": "object", - "properties": { - "questions": { - "type": "array", - "description": "The questions to be generated.", - "items": {"type": "string"}, - }, - }, - "required": ["questions"], - }, - }, - }, - ] - SYSTEM_PROMPT = "You are an AI assistant" - messages = [ - {"role": "system", "content": SYSTEM_PROMPT}, - { - "role": "user", - "content": "Generate 3 questions about civil engineering.", - }, - ] - try: - # test without max tokens - response = await acompletion( - model=model, - # model="claude-3-5-sonnet-20240620", - messages=messages, - stream=True, - temperature=0.75, - tools=tools, - stream_options={"include_usage": True}, - ) - idx = 0 - print(f"response: {response}") - async for chunk in response: - print(f"chunk in test: {chunk}") - if idx == 0: - assert ( - chunk.choices[0].delta.tool_calls[0].function.arguments is not None - ) - assert isinstance( - chunk.choices[0].delta.tool_calls[0].function.arguments, str - ) - validate_first_streaming_function_calling_chunk(chunk=chunk) - elif idx == 1 and chunk.choices[0].finish_reason is None: - validate_second_streaming_function_calling_chunk(chunk=chunk) - elif chunk.choices[0].finish_reason is not None: # last chunk - validate_final_streaming_function_calling_chunk(chunk=chunk) - idx += 1 - # raise Exception("it worked! ") - except litellm.InternalServerError as e: - pytest.skip(f"InternalServerError - {str(e)}") - except litellm.ServiceUnavailableError: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - - - - - - - - - - - - - - - - - -@pytest.mark.parametrize( - "model", - [ - "gpt-3.5-turbo", - "claude-sonnet-4-5-20250929", - "us.anthropic.claude-sonnet-4-5-20250929-v1:0", - # "vertex_ai/claude-3-5-sonnet@20240620", - ], -) -@pytest.mark.flaky(retries=3, delay=1) -def test_aastreaming_tool_calls_valid_json_str(model): - if "vertex_ai" in model: - from test_amazing_vertex_completion import ( - load_vertex_ai_credentials, - ) - - load_vertex_ai_credentials() - vertex_location = "us-east5" - else: - vertex_location = None - litellm.set_verbose = False - messages = [ - {"role": "user", "content": "Hit the snooze button."}, - ] - - tools = [ - { - "type": "function", - "function": { - "name": "snooze", - "parameters": { - "type": "object", - "properties": {}, - "required": [], - }, - }, - } - ] - - stream = litellm.completion( - model, messages, tools=tools, stream=True, vertex_location=vertex_location - ) - chunks = [*stream] - print(f"chunks: {chunks}") - tool_call_id_arg_map = {} - curr_tool_call_id = None - curr_tool_call_str = "" - for chunk in chunks: - if chunk.choices[0].delta.tool_calls is not None: - if chunk.choices[0].delta.tool_calls[0].id is not None: - # flush prev tool call - if curr_tool_call_id is not None: - tool_call_id_arg_map[curr_tool_call_id] = curr_tool_call_str - curr_tool_call_str = "" - curr_tool_call_id = chunk.choices[0].delta.tool_calls[0].id - tool_call_id_arg_map[curr_tool_call_id] = "" - if chunk.choices[0].delta.tool_calls[0].function.arguments is not None: - curr_tool_call_str += ( - chunk.choices[0].delta.tool_calls[0].function.arguments - ) - # flush prev tool call - if curr_tool_call_id is not None: - tool_call_id_arg_map[curr_tool_call_id] = curr_tool_call_str - - for k, v in tool_call_id_arg_map.items(): - print("k={}, v={}".format(k, v)) - json.loads(v) # valid json str - - -def test_streaming_api_base(): - litellm.set_verbose = False - stream = litellm.completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hey"}], - stream=True, - ) - assert "https://api.openai.com" in stream._hidden_params["api_base"] - - - - -@pytest.mark.parametrize( - "model", - [ - # "deepseek/deepseek-reasoner", - # "anthropic/claude-3-7-sonnet-20250219", - "openrouter/anthropic/claude-sonnet-4.5", - ], -) -def test_reasoning_content_completion(model): - # litellm.set_verbose = True - try: - # litellm.turn_on_debug() - resp = litellm.completion( - model=model, - messages=[{"role": "user", "content": "Tell me a joke."}], - stream=True, - # thinking={"type": "enabled", "budget_tokens": 1024}, - reasoning={"effort": "high"}, - drop_params=True, - ) - - reasoning_content_exists = False - for chunk in resp: - print(f"chunk 2: {chunk}") - if ( - hasattr(chunk.choices[0].delta, "reasoning_content") - and chunk.choices[0].delta.reasoning_content is not None - ): - reasoning_content_exists = True - break - assert reasoning_content_exists - except litellm.Timeout: - pytest.skip("Model is timing out") - - - - -def test_streaming_with_cost_calculation(): - from typing import Optional - - from litellm.types.utils import Usage - - litellm.include_cost_in_streaming_usage = True - - ## Test 1: check if usage object can handle 'cost' field - usage_object = Usage( - prompt_tokens=100, - completion_tokens=100, - total_tokens=200, - cost=1.0, - ) - assert usage_object.cost is not None - - print(f"usage_object: {usage_object}") - - ## Test 2: check if usage object has 'cost' field when streaming - - response = litellm.completion( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "What is the capital of France?"}], - stream=True, - stream_options={"include_usage": True}, - ) - - usage_object: Optional[Usage] = None - for chunk in response: - _usage_obj = getattr(chunk, "usage", None) - if _usage_obj is not None: - usage_object = _usage_obj - break - - assert usage_object is not None - assert usage_object.total_tokens is not None - assert usage_object.total_tokens > 0 - assert usage_object.prompt_tokens is not None - assert usage_object.prompt_tokens > 0 - assert usage_object.cost is not None - assert usage_object.cost > 0 - - -def test_streaming_finish_reason(): - litellm.set_verbose = False - - openai_finish_reason_idx: Optional[int] = None - openai_last_chunk_idx: Optional[int] = None - anthropic_finish_reason_idx: Optional[int] = None - anthropic_last_chunk_idx: Optional[int] = None - - ## OpenAI - response = litellm.completion( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "What is the capital of France?"}], - stream=True, - stream_options={"include_usage": True}, - ) - for idx, chunk in enumerate(response): - print(f"OPENAI CHUNK: {chunk}") - if chunk.choices[0].finish_reason is not None: - openai_finish_reason_idx = idx - openai_last_chunk_idx = idx - - assert openai_finish_reason_idx is not None - assert openai_finish_reason_idx > 0 - - ## Anthropic - response = litellm.completion( - model="anthropic/claude-sonnet-4-5-20250929", - messages=[{"role": "user", "content": "What is the capital of France?"}], - stream=True, - stream_options={"include_usage": True}, - ) - for idx, chunk in enumerate(response): - print(f"ANTHROPIC CHUNK: {chunk}") - if chunk.choices[0].finish_reason is not None: - anthropic_finish_reason_idx = idx - anthropic_last_chunk_idx = idx - - assert anthropic_finish_reason_idx is not None - assert anthropic_finish_reason_idx > 0 - - relative_anthropic_idx = anthropic_finish_reason_idx - anthropic_last_chunk_idx - relative_openai_idx = openai_finish_reason_idx - openai_last_chunk_idx - assert relative_anthropic_idx == relative_openai_idx diff --git a/tests/local_testing/test_team_config.py b/tests/local_testing/test_team_config.py deleted file mode 100644 index 8a5f8c8407f..00000000000 --- a/tests/local_testing/test_team_config.py +++ /dev/null @@ -1,36 +0,0 @@ -# #### What this tests #### -# # This tests if setting team_config actually works -# import sys, os -# import traceback -# import pytest - -# sys.path.insert( -# 0, os.path.abspath("../..") -# ) # Adds the parent directory to the system path -# import litellm -# from litellm.proxy.proxy_server import ProxyConfig - - -# @pytest.mark.asyncio -# async def test_team_config(): -# litellm.default_team_settings = [ -# { -# "team_id": "my-special-team", -# "success_callback": ["langfuse"], -# "langfuse_public_key": "os.environ/LANGFUSE_PUB_KEY_2", -# "langfuse_secret": "os.environ/LANGFUSE_PRIVATE_KEY_2", -# } -# ] -# proxyconfig = ProxyConfig() - -# team_config = await proxyconfig.load_team_config(team_id="my-special-team") -# assert len(team_config) > 0 - -# data = { -# "model": "gpt-3.5-turbo", -# "messages": [{"role": "user", "content": "Hey, how's it going?"}], -# } -# team_config.pop("team_id") -# response = litellm.completion(**{**data, **team_config}) - -# print(f"response: {response}") diff --git a/tests/local_testing/test_text_completion.py b/tests/local_testing/test_text_completion.py deleted file mode 100644 index a117ca975aa..00000000000 --- a/tests/local_testing/test_text_completion.py +++ /dev/null @@ -1,2932 +0,0 @@ -import json -import os -from types import MappingProxyType -from typing import Final - -from dotenv import load_dotenv -from unittest.mock import patch - -load_dotenv() -import pytest - -import litellm -from litellm import ( - TextCompletionResponse, - text_completion, -) - -litellm.num_retries = 3 - - -FIREWORKS_TEXT_COMPLETION: Final = MappingProxyType( - { - "model": "text-completion-openai/accounts/fireworks/models/glm-5p3-flash", - "api_base": "https://api.fireworks.ai/inference/v1", - "api_key": os.environ.get("FIREWORKS_AI_API_KEY"), - } -) - -token_prompt = [ - [ - 32, - 2043, - 32, - 329, - 4585, - 262, - 1644, - 14, - 34, - 3705, - 319, - 616, - 47551, - 30, - 930, - 19219, - 284, - 1949, - 284, - 787, - 428, - 355, - 1790, - 355, - 1744, - 981, - 1390, - 3307, - 2622, - 13, - 220, - 198, - 198, - 40, - 423, - 587, - 351, - 616, - 41668, - 32682, - 329, - 718, - 812, - 13, - 376, - 666, - 32682, - 468, - 281, - 4697, - 6621, - 11, - 356, - 1183, - 869, - 607, - 25737, - 11, - 508, - 318, - 2579, - 290, - 468, - 257, - 642, - 614, - 1468, - 1200, - 13, - 314, - 373, - 612, - 262, - 1110, - 25737, - 373, - 287, - 4827, - 290, - 14801, - 373, - 4642, - 11, - 673, - 318, - 616, - 41803, - 13, - 2399, - 2104, - 1641, - 468, - 6412, - 284, - 502, - 355, - 465, - 38074, - 494, - 1201, - 1110, - 352, - 13, - 314, - 716, - 407, - 2910, - 475, - 356, - 389, - 1641, - 11, - 673, - 3848, - 502, - 38074, - 494, - 290, - 356, - 423, - 3993, - 13801, - 11, - 26626, - 11864, - 11, - 3503, - 13, - 220, - 198, - 198, - 17, - 812, - 2084, - 25737, - 373, - 287, - 14321, - 422, - 2563, - 13230, - 13, - 21051, - 11, - 2356, - 25542, - 11, - 290, - 47482, - 897, - 547, - 607, - 1517, - 13, - 1375, - 550, - 257, - 5110, - 14608, - 290, - 262, - 1641, - 7723, - 1637, - 284, - 3758, - 607, - 284, - 14321, - 290, - 477, - 8389, - 257, - 7269, - 284, - 1011, - 1337, - 286, - 14801, - 13, - 383, - 5156, - 338, - 9955, - 11, - 25737, - 338, - 13850, - 11, - 468, - 257, - 47973, - 14, - 9979, - 2762, - 1693, - 290, - 373, - 503, - 286, - 3240, - 329, - 362, - 1933, - 523, - 339, - 2492, - 470, - 612, - 329, - 477, - 286, - 428, - 13, - 220, - 198, - 198, - 3347, - 10667, - 5223, - 503, - 706, - 513, - 1528, - 11, - 23630, - 673, - 373, - 366, - 38125, - 290, - 655, - 2622, - 257, - 3338, - 8399, - 1911, - 314, - 2298, - 607, - 510, - 11, - 1011, - 607, - 284, - 607, - 2156, - 11, - 290, - 673, - 3393, - 2925, - 284, - 7523, - 20349, - 290, - 4144, - 257, - 6099, - 13, - 314, - 836, - 470, - 892, - 20349, - 318, - 257, - 2563, - 290, - 716, - 845, - 386, - 12, - 66, - 1236, - 571, - 292, - 3584, - 314, - 836, - 470, - 7523, - 11, - 475, - 326, - 373, - 407, - 5035, - 6402, - 314, - 655, - 6497, - 607, - 510, - 422, - 14321, - 13, - 220, - 198, - 198, - 32, - 1285, - 1568, - 673, - 373, - 6294, - 329, - 3013, - 24707, - 287, - 262, - 12436, - 1539, - 819, - 5722, - 329, - 852, - 604, - 1933, - 2739, - 11, - 39398, - 607, - 1097, - 5059, - 981, - 1029, - 290, - 318, - 852, - 16334, - 329, - 720, - 1120, - 74, - 422, - 15228, - 278, - 656, - 257, - 2156, - 11, - 290, - 373, - 12165, - 503, - 286, - 376, - 666, - 32682, - 338, - 584, - 6621, - 338, - 2156, - 329, - 32012, - 262, - 14595, - 373, - 30601, - 510, - 290, - 2491, - 357, - 7091, - 373, - 1029, - 8, - 290, - 262, - 2104, - 34624, - 373, - 46432, - 1268, - 1961, - 422, - 1660, - 2465, - 780, - 8168, - 2073, - 1625, - 1363, - 329, - 807, - 2250, - 13, - 720, - 1238, - 11, - 830, - 286, - 2465, - 290, - 5875, - 5770, - 511, - 2156, - 5096, - 5017, - 340, - 13, - 220, - 198, - 198, - 2504, - 373, - 477, - 938, - 614, - 13, - 1119, - 1053, - 587, - 287, - 511, - 649, - 2156, - 319, - 511, - 898, - 329, - 546, - 718, - 1933, - 13, - 554, - 3389, - 673, - 1444, - 34020, - 290, - 531, - 511, - 8744, - 373, - 4423, - 572, - 780, - 673, - 1422, - 470, - 423, - 262, - 1637, - 780, - 41646, - 338, - 37751, - 1392, - 32621, - 510, - 290, - 1422, - 470, - 467, - 832, - 13, - 679, - 3432, - 511, - 2739, - 8744, - 9024, - 492, - 257, - 2472, - 286, - 720, - 4059, - 13, - 314, - 1807, - 340, - 373, - 13678, - 306, - 5789, - 475, - 4030, - 616, - 5422, - 4423, - 13, - 1439, - 468, - 587, - 5897, - 1201, - 13, - 220, - 198, - 198, - 7571, - 2745, - 2084, - 11, - 673, - 1965, - 502, - 284, - 8804, - 617, - 1637, - 284, - 651, - 38464, - 329, - 399, - 8535, - 13, - 3226, - 1781, - 314, - 1101, - 407, - 1016, - 284, - 1309, - 616, - 41803, - 393, - 6621, - 467, - 14720, - 11, - 645, - 2300, - 644, - 318, - 1016, - 319, - 4306, - 11, - 523, - 314, - 910, - 314, - 1183, - 307, - 625, - 379, - 642, - 13, - 314, - 1392, - 572, - 670, - 1903, - 290, - 651, - 612, - 379, - 362, - 25, - 2231, - 13, - 314, - 1282, - 287, - 1262, - 616, - 13952, - 1994, - 11, - 2513, - 287, - 11, - 766, - 399, - 8535, - 2712, - 351, - 36062, - 287, - 262, - 5228, - 11, - 25737, - 3804, - 503, - 319, - 262, - 18507, - 11, - 290, - 16914, - 319, - 262, - 6891, - 3084, - 13, - 8989, - 2406, - 422, - 257, - 1641, - 47655, - 351, - 13230, - 11, - 314, - 760, - 644, - 16914, - 3073, - 588, - 13, - 314, - 836, - 470, - 760, - 703, - 881, - 340, - 373, - 11, - 475, - 314, - 714, - 423, - 23529, - 276, - 340, - 510, - 290, - 5901, - 616, - 18057, - 351, - 340, - 13, - 314, - 6810, - 19772, - 2024, - 8347, - 287, - 262, - 2166, - 2119, - 290, - 399, - 8535, - 373, - 287, - 3294, - 11685, - 286, - 8242, - 290, - 607, - 7374, - 15224, - 13, - 383, - 4894, - 373, - 572, - 13, - 383, - 2156, - 373, - 3863, - 2319, - 37, - 532, - 340, - 373, - 1542, - 2354, - 13, - 220, - 198, - 198, - 40, - 1718, - 399, - 8535, - 284, - 616, - 1097, - 11, - 290, - 1444, - 16679, - 329, - 281, - 22536, - 355, - 314, - 373, - 12008, - 25737, - 373, - 14904, - 2752, - 13, - 220, - 314, - 1422, - 470, - 765, - 284, - 10436, - 290, - 22601, - 503, - 399, - 8535, - 523, - 314, - 9658, - 287, - 262, - 1097, - 290, - 1309, - 607, - 711, - 319, - 616, - 3072, - 1566, - 262, - 22536, - 5284, - 13, - 3226, - 1781, - 1644, - 290, - 32084, - 3751, - 510, - 355, - 880, - 13, - 314, - 4893, - 262, - 3074, - 290, - 780, - 399, - 8535, - 338, - 9955, - 318, - 503, - 286, - 3240, - 1762, - 11, - 34020, - 14, - 44, - 4146, - 547, - 1444, - 13, - 1649, - 484, - 5284, - 484, - 547, - 5897, - 290, - 4692, - 11, - 1422, - 470, - 1107, - 1561, - 11, - 1718, - 399, - 8535, - 11, - 290, - 1297, - 502, - 284, - 467, - 1363, - 13, - 220, - 198, - 198, - 2025, - 1711, - 1568, - 314, - 651, - 1363, - 290, - 41668, - 32682, - 7893, - 502, - 644, - 314, - 1053, - 1760, - 13, - 314, - 4893, - 2279, - 284, - 683, - 290, - 477, - 339, - 550, - 373, - 8993, - 329, - 502, - 13, - 18626, - 262, - 2104, - 1641, - 1541, - 2993, - 290, - 547, - 28674, - 379, - 502, - 329, - 644, - 314, - 550, - 1760, - 13, - 18626, - 314, - 373, - 366, - 448, - 286, - 1627, - 290, - 8531, - 1, - 780, - 314, - 1444, - 16679, - 878, - 4379, - 611, - 673, - 373, - 1682, - 31245, - 6, - 278, - 780, - 340, - 2900, - 503, - 673, - 373, - 655, - 47583, - 503, - 422, - 262, - 16914, - 13, - 775, - 8350, - 329, - 2250, - 290, - 314, - 1364, - 290, - 3377, - 262, - 1755, - 379, - 616, - 1266, - 1545, - 338, - 2156, - 290, - 16896, - 477, - 1755, - 13, - 314, - 3521, - 470, - 5412, - 340, - 477, - 523, - 314, - 2900, - 616, - 3072, - 572, - 290, - 3088, - 284, - 8960, - 290, - 655, - 9480, - 866, - 13, - 2011, - 1266, - 1545, - 373, - 510, - 477, - 1755, - 351, - 502, - 11, - 5149, - 502, - 314, - 750, - 2147, - 2642, - 11, - 290, - 314, - 1101, - 8788, - 13, - 220, - 198, - 198, - 40, - 1210, - 616, - 3072, - 319, - 290, - 314, - 550, - 6135, - 13399, - 14, - 37348, - 1095, - 13, - 31515, - 11, - 34020, - 11, - 47551, - 11, - 41668, - 32682, - 11, - 290, - 511, - 7083, - 1641, - 1866, - 24630, - 502, - 13, - 1119, - 389, - 2282, - 314, - 20484, - 607, - 1204, - 11, - 20484, - 399, - 8535, - 338, - 1204, - 11, - 925, - 2279, - 517, - 8253, - 621, - 340, - 2622, - 284, - 307, - 11, - 925, - 340, - 1171, - 618, - 340, - 373, - 257, - 366, - 17989, - 14669, - 1600, - 290, - 20484, - 25737, - 338, - 8395, - 286, - 1683, - 1972, - 20750, - 393, - 1719, - 10804, - 286, - 607, - 1200, - 757, - 11, - 4844, - 286, - 606, - 1683, - 765, - 284, - 766, - 502, - 757, - 290, - 314, - 481, - 1239, - 766, - 616, - 41803, - 757, - 11, - 290, - 484, - 765, - 502, - 284, - 1414, - 329, - 25737, - 338, - 7356, - 6314, - 290, - 20889, - 502, - 329, - 262, - 32084, - 1339, - 290, - 7016, - 12616, - 13, - 198, - 198, - 40, - 716, - 635, - 783, - 2060, - 13, - 1406, - 319, - 1353, - 286, - 6078, - 616, - 1266, - 1545, - 286, - 838, - 812, - 357, - 69, - 666, - 32682, - 828, - 314, - 481, - 4425, - 616, - 7962, - 314, - 550, - 351, - 683, - 11, - 644, - 314, - 3177, - 616, - 1641, - 11, - 290, - 616, - 399, - 8535, - 13, - 198, - 198, - 40, - 4988, - 1254, - 12361, - 13, - 314, - 423, - 12361, - 9751, - 284, - 262, - 966, - 810, - 314, - 1101, - 7960, - 2130, - 318, - 1016, - 284, - 1282, - 651, - 366, - 260, - 18674, - 1, - 319, - 502, - 329, - 644, - 314, - 750, - 13, - 314, - 460, - 470, - 4483, - 13, - 314, - 423, - 2626, - 767, - 8059, - 422, - 340, - 13, - 314, - 1101, - 407, - 11029, - 329, - 7510, - 13, - 314, - 423, - 11668, - 739, - 616, - 2951, - 13, - 314, - 1053, - 550, - 807, - 50082, - 12, - 12545, - 287, - 734, - 2745, - 13, - 1629, - 717, - 314, - 2936, - 523, - 6563, - 287, - 616, - 2551, - 475, - 355, - 262, - 1528, - 467, - 416, - 314, - 1101, - 3612, - 3863, - 484, - 547, - 826, - 290, - 314, - 815, - 423, - 10667, - 319, - 607, - 878, - 4585, - 16679, - 290, - 852, - 5306, - 3019, - 992, - 13, - 314, - 836, - 470, - 1337, - 546, - 25737, - 7471, - 11, - 475, - 314, - 750, - 18344, - 257, - 642, - 614, - 1468, - 1200, - 1497, - 422, - 607, - 3397, - 290, - 314, - 1254, - 12361, - 546, - 340, - 13, - 314, - 760, - 2130, - 287, - 262, - 1641, - 481, - 1011, - 607, - 287, - 11, - 475, - 340, - 338, - 1239, - 588, - 852, - 351, - 534, - 3397, - 13, - 1375, - 481, - 1663, - 510, - 20315, - 278, - 502, - 329, - 340, - 290, - 477, - 314, - 1053, - 1683, - 1760, - 318, - 1842, - 607, - 355, - 616, - 898, - 13, - 220, - 198, - 198, - 22367, - 11, - 317, - 2043, - 32, - 30, - 4222, - 1037, - 502, - 13, - 383, - 14934, - 318, - 6600, - 502, - 6776, - 13, - 220, - 198, - 24361, - 25, - 1148, - 428, - 2642, - 30, - 198, - 33706, - 25, - 645, - ], - [ - 32, - 2043, - 32, - 329, - 4585, - 262, - 1644, - 14, - 34, - 3705, - 319, - 616, - 47551, - 30, - 930, - 19219, - 284, - 1949, - 284, - 787, - 428, - 355, - 1790, - 355, - 1744, - 981, - 1390, - 3307, - 2622, - 13, - 220, - 198, - 198, - 40, - 423, - 587, - 351, - 616, - 41668, - 32682, - 329, - 718, - 812, - 13, - 376, - 666, - 32682, - 468, - 281, - 4697, - 6621, - 11, - 356, - 1183, - 869, - 607, - 25737, - 11, - 508, - 318, - 2579, - 290, - 468, - 257, - 642, - 614, - 1468, - 1200, - 13, - 314, - 373, - 612, - 262, - 1110, - 25737, - 373, - 287, - 4827, - 290, - 14801, - 373, - 4642, - 11, - 673, - 318, - 616, - 41803, - 13, - 2399, - 2104, - 1641, - 468, - 6412, - 284, - 502, - 355, - 465, - 38074, - 494, - 1201, - 1110, - 352, - 13, - 314, - 716, - 407, - 2910, - 475, - 356, - 389, - 1641, - 11, - 673, - 3848, - 502, - 38074, - 494, - 290, - 356, - 423, - 3993, - 13801, - 11, - 26626, - 11864, - 11, - 3503, - 13, - 220, - 198, - 198, - 17, - 812, - 2084, - 25737, - 373, - 287, - 14321, - 422, - 2563, - 13230, - 13, - 21051, - 11, - 2356, - 25542, - 11, - 290, - 47482, - 897, - 547, - 607, - 1517, - 13, - 1375, - 550, - 257, - 5110, - 14608, - 290, - 262, - 1641, - 7723, - 1637, - 284, - 3758, - 607, - 284, - 14321, - 290, - 477, - 8389, - 257, - 7269, - 284, - 1011, - 1337, - 286, - 14801, - 13, - 383, - 5156, - 338, - 9955, - 11, - 25737, - 338, - 13850, - 11, - 468, - 257, - 47973, - 14, - 9979, - 2762, - 1693, - 290, - 373, - 503, - 286, - 3240, - 329, - 362, - 1933, - 523, - 339, - 2492, - 470, - 612, - 329, - 477, - 286, - 428, - 13, - 220, - 198, - 198, - 3347, - 10667, - 5223, - 503, - 706, - 513, - 1528, - 11, - 23630, - 673, - 373, - 366, - 38125, - 290, - 655, - 2622, - 257, - 3338, - 8399, - 1911, - 314, - 2298, - 607, - 510, - 11, - 1011, - 607, - 284, - 607, - 2156, - 11, - 290, - 673, - 3393, - 2925, - 284, - 7523, - 20349, - 290, - 4144, - 257, - 6099, - 13, - 314, - 836, - 470, - 892, - 20349, - 318, - 257, - 2563, - 290, - 716, - 845, - 386, - 12, - 66, - 1236, - 571, - 292, - 3584, - 314, - 836, - 470, - 7523, - 11, - 475, - 326, - 373, - 407, - 5035, - 6402, - 314, - 655, - 6497, - 607, - 510, - 422, - 14321, - 13, - 220, - 198, - 198, - 32, - 1285, - 1568, - 673, - 373, - 6294, - 329, - 3013, - 24707, - 287, - 262, - 12436, - 1539, - 819, - 5722, - 329, - 852, - 604, - 1933, - 2739, - 11, - 39398, - 607, - 1097, - 5059, - 981, - 1029, - 290, - 318, - 852, - 16334, - 329, - 720, - 1120, - 74, - 422, - 15228, - 278, - 656, - 257, - 2156, - 11, - 290, - 373, - 12165, - 503, - 286, - 376, - 666, - 32682, - 338, - 584, - 6621, - 338, - 2156, - 329, - 32012, - 262, - 14595, - 373, - 30601, - 510, - 290, - 2491, - 357, - 7091, - 373, - 1029, - 8, - 290, - 262, - 2104, - 34624, - 373, - 46432, - 1268, - 1961, - 422, - 1660, - 2465, - 780, - 8168, - 2073, - 1625, - 1363, - 329, - 807, - 2250, - 13, - 720, - 1238, - 11, - 830, - 286, - 2465, - 290, - 5875, - 5770, - 511, - 2156, - 5096, - 5017, - 340, - 13, - 220, - 198, - 198, - 2504, - 373, - 477, - 938, - 614, - 13, - 1119, - 1053, - 587, - 287, - 511, - 649, - 2156, - 319, - 511, - 898, - 329, - 546, - 718, - 1933, - 13, - 554, - 3389, - 673, - 1444, - 34020, - 290, - 531, - 511, - 8744, - 373, - 4423, - 572, - 780, - 673, - 1422, - 470, - 423, - 262, - 1637, - 780, - 41646, - 338, - 37751, - 1392, - 32621, - 510, - 290, - 1422, - 470, - 467, - 832, - 13, - 679, - 3432, - 511, - 2739, - 8744, - 9024, - 492, - 257, - 2472, - 286, - 720, - 4059, - 13, - 314, - 1807, - 340, - 373, - 13678, - 306, - 5789, - 475, - 4030, - 616, - 5422, - 4423, - 13, - 1439, - 468, - 587, - 5897, - 1201, - 13, - 220, - 198, - 198, - 7571, - 2745, - 2084, - 11, - 673, - 1965, - 502, - 284, - 8804, - 617, - 1637, - 284, - 651, - 38464, - 329, - 399, - 8535, - 13, - 3226, - 1781, - 314, - 1101, - 407, - 1016, - 284, - 1309, - 616, - 41803, - 393, - 6621, - 467, - 14720, - 11, - 645, - 2300, - 644, - 318, - 1016, - 319, - 4306, - 11, - 523, - 314, - 910, - 314, - 1183, - 307, - 625, - 379, - 642, - 13, - 314, - 1392, - 572, - 670, - 1903, - 290, - 651, - 612, - 379, - 362, - 25, - 2231, - 13, - 314, - 1282, - 287, - 1262, - 616, - 13952, - 1994, - 11, - 2513, - 287, - 11, - 766, - 399, - 8535, - 2712, - 351, - 36062, - 287, - 262, - 5228, - 11, - 25737, - 3804, - 503, - 319, - 262, - 18507, - 11, - 290, - 16914, - 319, - 262, - 6891, - 3084, - 13, - 8989, - 2406, - 422, - 257, - 1641, - 47655, - 351, - 13230, - 11, - 314, - 760, - 644, - 16914, - 3073, - 588, - 13, - 314, - 836, - 470, - 760, - 703, - 881, - 340, - 373, - 11, - 475, - 314, - 714, - 423, - 23529, - 276, - 340, - 510, - 290, - 5901, - 616, - 18057, - 351, - 340, - 13, - 314, - 6810, - 19772, - 2024, - 8347, - 287, - 262, - 2166, - 2119, - 290, - 399, - 8535, - 373, - 287, - 3294, - 11685, - 286, - 8242, - 290, - 607, - 7374, - 15224, - 13, - 383, - 4894, - 373, - 572, - 13, - 383, - 2156, - 373, - 3863, - 2319, - 37, - 532, - 340, - 373, - 1542, - 2354, - 13, - 220, - 198, - 198, - 40, - 1718, - 399, - 8535, - 284, - 616, - 1097, - 11, - 290, - 1444, - 16679, - 329, - 281, - 22536, - 355, - 314, - 373, - 12008, - 25737, - 373, - 14904, - 2752, - 13, - 220, - 314, - 1422, - 470, - 765, - 284, - 10436, - 290, - 22601, - 503, - 399, - 8535, - 523, - 314, - 9658, - 287, - 262, - 1097, - 290, - 1309, - 607, - 711, - 319, - 616, - 3072, - 1566, - 262, - 22536, - 5284, - 13, - 3226, - 1781, - 1644, - 290, - 32084, - 3751, - 510, - 355, - 880, - 13, - 314, - 4893, - 262, - 3074, - 290, - 780, - 399, - 8535, - 338, - 9955, - 318, - 503, - 286, - 3240, - 1762, - 11, - 34020, - 14, - 44, - 4146, - 547, - 1444, - 13, - 1649, - 484, - 5284, - 484, - 547, - 5897, - 290, - 4692, - 11, - 1422, - 470, - 1107, - 1561, - 11, - 1718, - 399, - 8535, - 11, - 290, - 1297, - 502, - 284, - 467, - 1363, - 13, - 220, - 198, - 198, - 2025, - 1711, - 1568, - 314, - 651, - 1363, - 290, - 41668, - 32682, - 7893, - 502, - 644, - 314, - 1053, - 1760, - 13, - 314, - 4893, - 2279, - 284, - 683, - 290, - 477, - 339, - 550, - 373, - 8993, - 329, - 502, - 13, - 18626, - 262, - 2104, - 1641, - 1541, - 2993, - 290, - 547, - 28674, - 379, - 502, - 329, - 644, - 314, - 550, - 1760, - 13, - 18626, - 314, - 373, - 366, - 448, - 286, - 1627, - 290, - 8531, - 1, - 780, - 314, - 1444, - 16679, - 878, - 4379, - 611, - 673, - 373, - 1682, - 31245, - 6, - 278, - 780, - 340, - 2900, - 503, - 673, - 373, - 655, - 47583, - 503, - 422, - 262, - 16914, - 13, - 775, - 8350, - 329, - 2250, - 290, - 314, - 1364, - 290, - 3377, - 262, - 1755, - 379, - 616, - 1266, - 1545, - 338, - 2156, - 290, - 16896, - 477, - 1755, - 13, - 314, - 3521, - 470, - 5412, - 340, - 477, - 523, - 314, - 2900, - 616, - 3072, - 572, - 290, - 3088, - 284, - 8960, - 290, - 655, - 9480, - 866, - 13, - 2011, - 1266, - 1545, - 373, - 510, - 477, - 1755, - 351, - 502, - 11, - 5149, - 502, - 314, - 750, - 2147, - 2642, - 11, - 290, - 314, - 1101, - 8788, - 13, - 220, - 198, - 198, - 40, - 1210, - 616, - 3072, - 319, - 290, - 314, - 550, - 6135, - 13399, - 14, - 37348, - 1095, - 13, - 31515, - 11, - 34020, - 11, - 47551, - 11, - 41668, - 32682, - 11, - 290, - 511, - 7083, - 1641, - 1866, - 24630, - 502, - 13, - 1119, - 389, - 2282, - 314, - 20484, - 607, - 1204, - 11, - 20484, - 399, - 8535, - 338, - 1204, - 11, - 925, - 2279, - 517, - 8253, - 621, - 340, - 2622, - 284, - 307, - 11, - 925, - 340, - 1171, - 618, - 340, - 373, - 257, - 366, - 17989, - 14669, - 1600, - 290, - 20484, - 25737, - 338, - 8395, - 286, - 1683, - 1972, - 20750, - 393, - 1719, - 10804, - 286, - 607, - 1200, - 757, - 11, - 4844, - 286, - 606, - 1683, - 765, - 284, - 766, - 502, - 757, - 290, - 314, - 481, - 1239, - 766, - 616, - 41803, - 757, - 11, - 290, - 484, - 765, - 502, - 284, - 1414, - 329, - 25737, - 338, - 7356, - 6314, - 290, - 20889, - 502, - 329, - 262, - 32084, - 1339, - 290, - 7016, - 12616, - 13, - 198, - 198, - 40, - 716, - 635, - 783, - 2060, - 13, - 1406, - 319, - 1353, - 286, - 6078, - 616, - 1266, - 1545, - 286, - 838, - 812, - 357, - 69, - 666, - 32682, - 828, - 314, - 481, - 4425, - 616, - 7962, - 314, - 550, - 351, - 683, - 11, - 644, - 314, - 3177, - 616, - 1641, - 11, - 290, - 616, - 399, - 8535, - 13, - 198, - 198, - 40, - 4988, - 1254, - 12361, - 13, - 314, - 423, - 12361, - 9751, - 284, - 262, - 966, - 810, - 314, - 1101, - 7960, - 2130, - 318, - 1016, - 284, - 1282, - 651, - 366, - 260, - 18674, - 1, - 319, - 502, - 329, - 644, - 314, - 750, - 13, - 314, - 460, - 470, - 4483, - 13, - 314, - 423, - 2626, - 767, - 8059, - 422, - 340, - 13, - 314, - 1101, - 407, - 11029, - 329, - 7510, - 13, - 314, - 423, - 11668, - 739, - 616, - 2951, - 13, - 314, - 1053, - 550, - 807, - 50082, - 12, - 12545, - 287, - 734, - 2745, - 13, - 1629, - 717, - 314, - 2936, - 523, - 6563, - 287, - 616, - 2551, - 475, - 355, - 262, - 1528, - 467, - 416, - 314, - 1101, - 3612, - 3863, - 484, - 547, - 826, - 290, - 314, - 815, - 423, - 10667, - 319, - 607, - 878, - 4585, - 16679, - 290, - 852, - 5306, - 3019, - 992, - 13, - 314, - 836, - 470, - 1337, - 546, - 25737, - 7471, - 11, - 475, - 314, - 750, - 18344, - 257, - 642, - 614, - 1468, - 1200, - 1497, - 422, - 607, - 3397, - 290, - 314, - 1254, - 12361, - 546, - 340, - 13, - 314, - 760, - 2130, - 287, - 262, - 1641, - 481, - 1011, - 607, - 287, - 11, - 475, - 340, - 338, - 1239, - 588, - 852, - 351, - 534, - 3397, - 13, - 1375, - 481, - 1663, - 510, - 20315, - 278, - 502, - 329, - 340, - 290, - 477, - 314, - 1053, - 1683, - 1760, - 318, - 1842, - 607, - 355, - 616, - 898, - 13, - 220, - 198, - 198, - 22367, - 11, - 317, - 2043, - 32, - 30, - 4222, - 1037, - 502, - 13, - 383, - 14934, - 318, - 6600, - 502, - 6776, - 13, - 220, - 198, - 24361, - 25, - 1148, - 428, - 2642, - 30, - 198, - 33706, - 25, - 3763, - ], -] - - -def test_completion_openai_prompt(): - try: - print("\n text 003 test\n") - response = text_completion( - prompt=["What's the weather in SF?", "How is Manchester?"], - max_tokens=5, - **FIREWORKS_TEXT_COMPLETION, - ) - print(response) - assert len(response.choices) == 2 - response_str = response["choices"][0]["text"] - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_completion_openai_prompt() - - -def test_completion_openai_engine_and_model() -> None: - response: Final = text_completion( - model="gpt-6-luna", - engine="anything", - reasoning_effort="none", - prompt="What's the weather in SF?", - max_tokens=5, - ) - assert response.model == "gpt-6-luna" - assert response.choices[0].text - - -# test_completion_openai_engine_and_model() - - -def test_completion_openai_engine() -> None: - response: Final = text_completion( - engine="gpt-6-luna", - reasoning_effort="none", - prompt="What's the weather in SF?", - max_tokens=5, - ) - assert response.model == "gpt-6-luna" - assert response.choices[0].text - - -# test_completion_openai_engine() - - -# test_completion_chatgpt_prompt() - - -def test_completion_gpt_instruct(): - try: - response = text_completion( - model="gpt-5.4-nano", - prompt="What's the weather in SF?", - custom_llm_provider="text-completion-openai", - ) - print(response) - response_str = response["choices"][0]["text"] - print("\n", response.choices) - print("\n", response.choices[0]) - # print(response.choices[0].text) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_completion_chatgpt_prompt() - - -def test_text_completion_basic(): - try: - print("\n test 003 with logprobs \n") - litellm.set_verbose = False - response = text_completion( - model="text-completion-openai/gpt-5.4-nano", - prompt="good morning", - max_tokens=10, - logprobs=10, - ) - print(response) - print(response.choices) - print(response.choices[0]) - # print(response.choices[0].text) - response_str = response["choices"][0]["text"] - except Exception as e: - if "502: Bad gateway" in str(e): - print("502: Bad gateway error occurred... passing") - return - pytest.fail(f"Error occurred: {e}") - - -# test_text_completion_basic() - - -def test_completion_text_003_prompt_array(): - try: - litellm.set_verbose = False - response = text_completion( - prompt=token_prompt, # token prompt is a 2d list - max_tokens=5, - **FIREWORKS_TEXT_COMPLETION, - ) - assert len(response.choices) == len(token_prompt) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - -# test_completion_text_003_prompt_array() - - -# not including this in our ci cd pipeline, since we don't want to fail tests due to an unstable replit -# def test_text_completion_with_proxy(): -# try: -# litellm.set_verbose=True -# response = text_completion( -# model="facebook/opt-125m", -# prompt='Write a tagline for a traditional bavarian tavern', -# api_base="https://openai-proxy.berriai.repl.co/v1", -# custom_llm_provider="openai", -# temperature=0, -# max_tokens=10, -# ) -# print("\n\n response") - -# print(response) -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") -# test_text_completion_with_proxy() - - -##### hugging face tests - - -# test_completion_hf_prompt_array() - - -# test_text_completion_stream() - -# async def test_text_completion_async_stream(): -# try: -# response = await atext_completion( -# model="text-completion-openai/gpt-3.5-turbo-instruct", -# prompt="good morning", -# stream=True, -# max_tokens=10, -# ) -# async for chunk in response: -# print(f"chunk: {chunk}") -# except Exception as e: -# pytest.fail(f"GOT exception for HF In streaming{e}") - -# asyncio.run(test_text_completion_async_stream()) - - - - -# test_async_text_completion() - - -@pytest.mark.asyncio -async def test_async_text_completion_stream() -> None: - response: Final = await litellm.atext_completion( - model="gpt-6-luna", - reasoning_effort="none", - prompt="good morning", - stream=True, - max_tokens=32, - ) - chunks: Final = [chunk async for chunk in response] - assert sum(chunk.choices[0].finish_reason is not None for chunk in chunks) == 1 - assert any(chunk.choices[0].text for chunk in chunks) - - -# test_async_text_completion_stream() - - -@pytest.mark.asyncio -async def test_async_text_completion_chat_model_stream(): - try: - response = await litellm.atext_completion( - model="gpt-3.5-turbo", - prompt="good morning", - stream=True, - max_tokens=10, - ) - - num_finish_reason = 0 - chunks = [] - async for chunk in response: - print(chunk) - chunks.append(chunk) - if chunk["choices"][0].get("finish_reason") is not None: - num_finish_reason += 1 - - assert num_finish_reason == 1, f"expected only one finish reason. Got {num_finish_reason}" - response_obj = litellm.stream_chunk_builder(chunks=chunks) - cost = litellm.completion_cost(completion_response=response_obj) - assert cost > 0 - except Exception as e: - pytest.fail(f"GOT exception for gpt-3.5 In streaming{e}") - - -# asyncio.run(test_async_text_completion_chat_model_stream()) - - -@pytest.mark.parametrize("stream", [True, False]) -def test_text_completion_with_echo(stream): - litellm.set_verbose = True - response = litellm.text_completion( - prompt="hello", - **FIREWORKS_TEXT_COMPLETION, - max_tokens=1, # only see the first token - stop="\n", # stop at the first newline - logprobs=1, # return log prob - echo=True, # if True, return the prompt as well - stream=stream, - ) - print(response) - - if stream: - for chunk in response: - print(chunk) - else: - assert isinstance(response, TextCompletionResponse) - assert response.choices[0].text.startswith("hello") - assert response.choices[0].logprobs.token_logprobs - - -def test_text_completion_ollama(): - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - client = HTTPHandler() - - with patch.object(client, "post") as mock_call: - try: - response = litellm.text_completion( - model="ollama/llama3.1:8b", - prompt="hello", - client=client, - ) - print(response) - except Exception as e: - print(e) - - mock_call.assert_called_once() - print(mock_call.call_args.kwargs) - json_data = json.loads(mock_call.call_args.kwargs["data"]) - assert json_data["prompt"] == "hello" diff --git a/tests/local_testing/test_together_ai.py b/tests/local_testing/test_together_ai.py deleted file mode 100644 index d4d5f968a54..00000000000 --- a/tests/local_testing/test_together_ai.py +++ /dev/null @@ -1,64 +0,0 @@ -# import sys, os -# import traceback - -# sys.path.insert( -# 0, os.path.abspath("../..") -# ) # Adds the parent directory to the system path -# import time -# import litellm -# import openai -# import pytest - -# ### Together AI -# import together -# together.api_key = "" - -# sample_message = [ -# {"role": "user", "content": "Who are you"}, -# {"role": "assistant", "content": "I am your helpful assistant."}, -# {"role": "user", "content": "Tell me a joke"}, -# ] - - -# def format_prompt_togetherai(messages, prompt_format, stop_words): -# start_token, end_token = prompt_format.split('{prompt}') -# prompt = '' -# for message in messages: -# role = message['role'] -# message_content = message['content'] -# if role == 'system': -# prompt += f"{start_token}\n<>\n{message_content}\n<>\n" -# elif role == 'user': -# prompt += f"{start_token}{message_content}{end_token}" -# else: -# prompt += f'{message_content}{stop_words[0]}' -# return prompt - - -# model = 'togethercomputer/CodeLlama-13b-Instruct' -# stop_words = list(together.Models.info(model)['config']['stop']) -# prompt_format = str(together.Models.info(model)['config']['prompt_format']) -# formatted_prompt = format_prompt_togetherai( -# messages=sample_message, prompt_format=prompt_format, stop_words=stop_words) -# for token in together.Complete.create_streaming(prompt=formatted_prompt, -# model=model, stop=stop_words, max_tokens=512): -# print(token, end="") - - -# ### litellm - -# import os -# from litellm import completion - -# os.environ["TOGETHERAI_API_KEY"] = "" - -# sample_message = [ -# {"role": "user", "content": "Who are you"}, -# {"role": "assistant", "content": "I am your helpful assistant."}, -# {"role": "user", "content": "Tell me a joke"}, -# ] - -# res = completion(model="together_ai/togethercomputer/CodeLlama-13b-Instruct", -# messages=sample_message, stream=False, max_tokens=1000) - -# print(list(res)) diff --git a/tests/local_testing/test_validate_environment.py b/tests/local_testing/test_validate_environment.py deleted file mode 100644 index 289c2bb7c99..00000000000 --- a/tests/local_testing/test_validate_environment.py +++ /dev/null @@ -1,10 +0,0 @@ -#### What this tests #### -# This tests the validate environment function - -import sys, os -import traceback - -import time -import litellm - -print(litellm.validate_environment("openai/gpt-3.5-turbo")) diff --git a/tests/unit/caching/test_caching.py b/tests/unit/caching/test_caching.py index 12dbcc9169a..0cad6e350e3 100644 --- a/tests/unit/caching/test_caching.py +++ b/tests/unit/caching/test_caching.py @@ -9,7 +9,9 @@ from datetime import timedelta from typing import Final from unittest.mock import AsyncMock, MagicMock, call, patch +import httpx import pytest +import respx import litellm from litellm import acompletion, aembedding, completion, embedding @@ -1027,6 +1029,47 @@ def test_completion_past_max_messages_is_neither_served_from_nor_written_to_the_ assert answer(five, "five second") == "five second", "a 5-message repeat was served from the cache" +@respx.mock +def test_enable_cache_serves_a_repeated_bedrock_titan_embedding_from_the_local_cache( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], fake_provider_credentials: None +) -> None: + monkeypatch.setattr(litellm, "cache", None) + monkeypatch.setattr(litellm, "set_verbose", True) + monkeypatch.setattr(litellm, "input_callback", []) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + route: Final = respx.post( + "https://bedrock-runtime.us-west-2.amazonaws.com/model/amazon.titan-embed-text-v2:0/invoke" + ).mock(return_value=httpx.Response(200, json={"embedding": [0.25, -0.5, 0.75], "inputTextTokenCount": 6})) + request: Final = { + "model": "bedrock/amazon.titan-embed-text-v2:0", + "input": "good morning from litellm, attempting to embed data", + "aws_region_name": "us-west-2", + } + + litellm.enable_cache() + enabled_cache: Final = litellm.cache + monkeypatch.setattr(litellm, "input_callback", []) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + litellm.enable_cache() + + first: Final = embedding(**request) + second: Final = embedding(**request) + + assert isinstance(enabled_cache, Cache) + assert enabled_cache.type == LiteLLMCacheType.LOCAL + assert litellm.cache is enabled_cache + assert litellm.input_callback == ["cache"] + assert litellm.success_callback == ["cache"] + assert litellm._async_success_callback == ["cache"] + assert "LiteLLM: Enabling Cache" in capsys.readouterr().out + assert route.call_count == 1 + assert first.data[0]["embedding"] == [0.25, -0.5, 0.75] + assert second.data[0]["embedding"] == [0.25, -0.5, 0.75] + assert second._hidden_params["cache_hit"] is True + + def test_basic_caching_import() -> None: from litellm.caching import Cache as PublicCache diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index bb7af3f1c06..9647ae07c35 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -6,7 +6,9 @@ import re from typing import Final, List from unittest.mock import MagicMock, patch +import httpx import pytest +import respx import litellm from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -22,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_messages_pt, convert_to_anthropic_tool_result, convert_to_gemini_tool_call_result, + convert_to_ollama_image, encode_tool_call_id_with_signature, function_call_prompt, get_thought_signature_from_tool, @@ -285,6 +288,23 @@ def test_ollama_pt_rejects_a_malformed_content_part_as_a_bad_request(part: objec assert expected_detail in excinfo.value.message +def test_convert_to_ollama_image_downloads_an_http_image_url_to_bare_base64(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "user_url_validation", False) + png_bytes: Final = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" + ) + image_url: Final = f"https://images.example.com/{uuid.uuid4()}.png" + + with respx.mock: + route: Final = respx.get(image_url).mock( + return_value=httpx.Response(200, content=png_bytes, headers={"content-type": "image/png"}) + ) + result: Final = convert_to_ollama_image(image_url) + + assert route.call_count == 1 + assert result == base64.b64encode(png_bytes).decode("utf-8") + + @pytest.mark.asyncio async def test_anthropic_bedrock_thinking_blocks_with_none_content(): """ diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 771ed5d2e15..f8ed11bdca3 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -3972,6 +3972,61 @@ async def test_async_success_handler_prevents_reprocessing_for_pass_through_endp ) +class _StreamSuccessRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.logged_contents: list[str | None] = [] + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.logged_contents.append(response_obj.choices[0].message.content) + + +@pytest.mark.asyncio +async def test_async_success_handler_logs_a_completed_stream_only_once(): + recorder: Final = _StreamSuccessRecorder() + start: Final = dt_object(2026, 1, 1) + logging_obj: Final = Logging( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="acompletion", + start_time=start, + litellm_call_id="stream-once", + function_id="stream-once", + dynamic_async_success_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model="gpt-4o", + user=None, + optional_params={}, + litellm_params={"litellm_call_id": "stream-once", "metadata": {}}, + custom_llm_provider="openai", + ) + + await logging_obj.async_success_handler( + result=ModelResponse( + model="gpt-4o", + choices=[{"index": 0, "message": {"role": "assistant", "content": "Hello there"}, "finish_reason": "stop"}], + usage={"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + ), + start_time=start, + end_time=start, + ) + await logging_obj.async_success_handler( + result=ModelResponse( + model="gpt-4o", + choices=[{"index": 0, "message": {"role": "assistant", "content": "Hello again"}, "finish_reason": "stop"}], + ), + start_time=start, + end_time=start, + ) + + assert recorder.logged_contents == ["Hello there"] + assert logging_obj.model_call_details["async_complete_streaming_response"].choices[0].message.content == ( + "Hello there" + ) + + @pytest.mark.asyncio async def test_async_success_handler_sets_standard_logging_object_for_streaming_pass_through(): """ diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index 83e53b2d80a..3f73eace59f 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1,10 +1,15 @@ +import base64 +import json from collections.abc import Mapping, Sequence from typing import Final from unittest.mock import MagicMock +import httpx import pytest +import respx +import litellm from litellm import ChatCompletionUsageBlock, stream_chunk_builder from litellm.types.utils import GenericStreamingChunk from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor @@ -1851,3 +1856,285 @@ def test_stream_chunk_builder_omits_service_tier_when_no_chunk_carried_one(): assert response is not None assert "service_tier" not in response.model_dump() + + +def _legacy_stream_chunk( + model: str, + *, + content: str | None = None, + tool_calls: tuple[ChatCompletionDeltaToolCall, ...] = (), + usage: Usage | None = None, + finish_reason: str | None = None, +) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-legacy", + created=1, + model=model, + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason=finish_reason, + index=0, + delta=Delta(content=content, role="assistant", tool_calls=list(tool_calls)), + ) + ], + usage=usage, + ) + + +def test_stream_chunk_builder_empty_initial_chunk() -> None: + chunks: Final = ({"id": ""}, {"id": "1"}, {"id": "1"}) + + assert ChunkProcessor._get_chunk_id(chunks) == "1" + + +def test_stream_chunk_builder_litellm_empty_chunks() -> None: + with pytest.raises(litellm.APIError): + stream_chunk_builder(chunks=None) + + assert stream_chunk_builder(chunks=[]) is None + + +def test_stream_chunk_builder_litellm_mixed_calls() -> None: + chunks: Final = ( + _legacy_stream_chunk( + "groq/openai/gpt-oss-120b", + content="To answer your question, ", + tool_calls=( + ChatCompletionDeltaToolCall( + id="call_sql", + index=0, + type="function", + function=Function( + name="sql_query", + arguments='{"query": "SELECT COUNT(*) FROM users;"}', + ), + ), + ), + ), + _legacy_stream_chunk( + "groq/openai/gpt-oss-120b", + content="I will run a query.", + finish_reason="stop", + ), + ) + + response: Final = stream_chunk_builder(chunks=chunks) + + assert response is not None + assert response.choices[0].message.content == "To answer your question, I will run a query." + assert [tool_call.to_dict() for tool_call in response.choices[0].message.tool_calls] == [ + { + "function": { + "arguments": '{"query": "SELECT COUNT(*) FROM users;"}', + "name": "sql_query", + }, + "id": "call_sql", + "type": "function", + } + ] + + +def _openai_sse(*chunks: Mapping[str, object]) -> str: + events: Final = tuple( + f"data: {json.dumps({'id': 'chatcmpl-legacy', 'object': 'chat.completion.chunk', 'created': 1, 'model': 'gpt-4.1-mini', **chunk})}\n\n" + for chunk in chunks + ) + return "".join(events) + "data: [DONE]\n\n" + + +def _complete_streamed_openai_response(respx_mock: respx.MockRouter, sse_body: str) -> litellm.ModelResponse: + respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, text=sse_body, headers={"content-type": "text/event-stream"}) + ) + return litellm.completion( + model="openai/gpt-4.1-mini", + messages=[{"role": "user", "content": "What's the weather like in Boston?"}], + api_key="sk-test", + stream=True, + stream_options={"include_usage": True}, + complete_response=True, + ) + + +def test_stream_chunk_builder_litellm_tool_call(respx_mock: respx.MockRouter) -> None: + response: Final = _complete_streamed_openai_response( + respx_mock, + _openai_sse( + { + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "id": "call_weather", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city":'}, + } + ], + }, + "finish_reason": None, + } + ] + }, + { + "choices": [ + { + "index": 0, + "delta": {"tool_calls": [{"index": 0, "function": {"arguments": '"Boston"}'}}]}, + "finish_reason": "tool_calls", + } + ] + }, + {"choices": [], "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}}, + ), + ) + + assert [tool_call.to_dict() for tool_call in response.choices[0].message.tool_calls] == [ + {"function": {"arguments": '{"city":"Boston"}', "name": "get_weather"}, "id": "call_weather", "type": "function"} + ] + assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (5, 3, 8) + + +def test_stream_chunk_builder_litellm_tool_call_regular_message(respx_mock: respx.MockRouter) -> None: + response: Final = _complete_streamed_openai_response( + respx_mock, + _openai_sse( + {"choices": [{"index": 0, "delta": {"role": "assistant", "content": "Hello "}, "finish_reason": None}]}, + {"choices": [{"index": 0, "delta": {"content": "there."}, "finish_reason": "stop"}]}, + {"choices": [], "usage": {"prompt_tokens": 4, "completion_tokens": 2, "total_tokens": 6}}, + ), + ) + + assert response.choices[0].message.content == "Hello there." + assert response.choices[0].message.tool_calls is None + assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == (4, 2, 6) + assert response._hidden_params["custom_llm_provider"] == "openai" + + +def test_stream_chunk_builder_multiple_tool_calls() -> None: + def tool_call_chunk(*tool_calls: ChatCompletionDeltaToolCall, finish_reason: str | None = None) -> ModelResponseStream: + return _legacy_stream_chunk("openai/gpt-4o", tool_calls=tool_calls, finish_reason=finish_reason) + + chunks: Final = ( + tool_call_chunk( + ChatCompletionDeltaToolCall( + id="call_exp", index=0, type="function", function=Function(name="exponentiate", arguments="") + ) + ), + tool_call_chunk(ChatCompletionDeltaToolCall(index=0, function=Function(arguments='{"base": 3, '))), + tool_call_chunk( + ChatCompletionDeltaToolCall(id="call_add", index=1, type="function", function=Function(name="add", arguments="")) + ), + tool_call_chunk(ChatCompletionDeltaToolCall(index=0, function=Function(arguments='"exponent": 5}'))), + tool_call_chunk( + ChatCompletionDeltaToolCall(index=1, function=Function(arguments='{"x": 1, "y": 2}')), + finish_reason="tool_calls", + ), + ) + + response: Final = stream_chunk_builder(chunks=chunks) + + assert response is not None + assert response.choices[0].finish_reason == "tool_calls" + assert [tool_call.to_dict() for tool_call in response.choices[0].message.tool_calls] == [ + {"function": {"arguments": '{"base": 3, "exponent": 5}', "name": "exponentiate"}, "id": "call_exp", "type": "function"}, + {"function": {"arguments": '{"x": 1, "y": 2}', "name": "add"}, "id": "call_add", "type": "function"}, + ] + + +def test_grok_bug() -> None: + chunks: Final = ( + _legacy_stream_chunk( + "groq/openai/gpt-oss-120b", + tool_calls=( + ChatCompletionDeltaToolCall( + id="call_weather", + index=0, + type="function", + function=Function(name="get_weather", arguments='{"location":'), + ), + ), + ), + _legacy_stream_chunk( + "groq/openai/gpt-oss-120b", + tool_calls=( + ChatCompletionDeltaToolCall( + index=0, + type="function", + function=Function(arguments='"Boston"}'), + ), + ), + finish_reason="tool_calls", + ), + ) + + response: Final = stream_chunk_builder(chunks=chunks) + + assert response is not None + assert json.loads(response.choices[0].message.tool_calls[0].function.arguments) == { + "location": "Boston" + } + + +def test_stream_chunk_builder_openai_audio_output_usage() -> None: + usage: Final = Usage( + completion_tokens=4, + prompt_tokens=7, + total_tokens=11, + completion_tokens_details={"audio_tokens": 2, "text_tokens": 2}, + prompt_tokens_details=PromptTokensDetails(audio_tokens=3, text_tokens=4), + ) + response: Final = stream_chunk_builder( + chunks=[ + _legacy_stream_chunk("openai/gpt-audio-1.5", content="Ye"), + _legacy_stream_chunk("openai/gpt-audio-1.5", content="s.", finish_reason="stop"), + _legacy_stream_chunk("openai/gpt-audio-1.5", usage=usage), + ] + ) + + assert response is not None + assert response.choices[0].message.content == "Yes." + assert response.usage.model_dump(exclude_none=True) == usage.model_dump(exclude_none=True) + + +def _openai_audio_stream_chunk(delta: Mapping[str, object], finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + **{ + "id": "chatcmpl-audio", + "created": 1, + "model": "gpt-audio-1.5", + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": dict(delta), "finish_reason": finish_reason, "logprobs": None}], + } + ) + + +def test_stream_chunk_builder_joins_openai_audio_output_pieces() -> None: + first_pcm: Final = b"\x01\x02" + second_pcm: Final = b"\x03\x04\x05\x06" + chunks: Final = ( + _openai_audio_stream_chunk( + {"role": "assistant", "content": None, "audio": {"id": "audio_yes", "transcript": "Ye"}} + ), + _openai_audio_stream_chunk({"audio": {"transcript": "s."}}), + _openai_audio_stream_chunk({"audio": {"data": base64.b64encode(first_pcm).decode()}}), + _openai_audio_stream_chunk({"audio": {"data": base64.b64encode(second_pcm).decode()}}), + _openai_audio_stream_chunk({"audio": {"id": "audio_yes", "expires_at": 1767225600}}), + _openai_audio_stream_chunk({}, finish_reason="stop"), + ) + + response: Final = stream_chunk_builder(chunks=list(chunks)) + + assert response is not None + audio: Final = response.choices[0].message.audio + assert audio is not None + assert audio.model_dump() == { + "id": "audio_yes", + "data": base64.b64encode(first_pcm + second_pcm).decode(), + "expires_at": 1767225600, + "transcript": "Yes.", + } diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index 6d7d62e47aa..e33780e4cd9 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -1,11 +1,13 @@ import json import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from itertools import chain from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +import respx import litellm from litellm._uuid import uuid @@ -18,6 +20,7 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallFunctionChunk, ) from litellm.types.responses.main import OutputCodeInterpreterCall +from litellm.types.utils import ChatCompletionDeltaToolCall, ModelResponseStream @pytest.mark.asyncio @@ -86,6 +89,120 @@ def test_anthropic_completion_does_not_send_deployment_default_limits(): assert "default_api_key_tpm_limit" not in request_body +def test_anthropic_tool_history_without_tools_injects_a_dummy_tool( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "modify_params", False) + route: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response( + 200, + json={ + "id": "msg_tool_history", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5-20251001", + "content": [{"type": "text", "text": "San Francisco is 72 degrees"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 4, "output_tokens": 6}, + }, + ) + ) + + response: Final = litellm.completion( + model="anthropic/claude-haiku-4-5-20251001", + messages=[ + {"role": "user", "content": "What's the weather like in San Francisco?"}, + { + "role": "assistant", + "content": "Here are the current weather conditions", + "tool_calls": [ + { + "id": "tooluse_weather", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": '{"location": "San Francisco, CA", "unit": "fahrenheit"}', + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "tooluse_weather", + "name": "get_current_weather", + "content": '{"temperature": "72"}', + }, + ], + api_key="anthropic-test-key", + max_tokens=32, + ) + + request_body: Final = json.loads(route.calls.last.request.read()) + assert request_body["tools"] == [ + { + "name": "dummy_tool", + "input_schema": {"type": "object", "properties": {}}, + "type": "custom", + "description": "This is a dummy tool call", + } + ] + assert request_body["messages"][1:] == [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Here are the current weather conditions"}, + { + "type": "tool_use", + "id": "tooluse_weather", + "name": "get_current_weather", + "input": {"location": "San Francisco, CA", "unit": "fahrenheit"}, + }, + ], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "tooluse_weather", "content": '{"temperature": "72"}'}], + }, + ] + assert response.choices[0].message.content == "San Francisco is 72 degrees" + + +@respx.mock +def test_anthropic_tool_call_error_is_mapped_from_the_provider_response(): + route = respx.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response( + 400, + json={ + "type": "error", + "error": {"type": "invalid_request_error", "message": "tool input is invalid"}, + }, + ) + ) + + with pytest.raises(litellm.BadRequestError) as exc_info: + litellm.completion( + model="anthropic/claude-3-5-haiku-20241022", + messages=[{"role": "user", "content": "call the tool"}], + tools=[ + { + "type": "function", + "function": { + "name": "lookup", + "description": "look up a value", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + api_key="anthropic-test-key", + ) + + assert exc_info.value.status_code == 400 + assert exc_info.value.llm_provider == "anthropic" + assert "tool input is invalid" in str(exc_info.value) + assert len(route.calls) == 1 + + async def test_anthropic_async_completion_inlines_http_images_off_the_event_loop(async_only_image_fetch): http_image_url = f"http://img.example/{uuid.uuid4()}.png" https_image_url = f"https://img.example/{uuid.uuid4()}.png" @@ -592,6 +709,172 @@ def test_message_delta_without_usage_returns_chunk_with_no_usage(): assert model_response.usage is None +async def _collect_anthropic_stream(sync_mode: bool, **kwargs: object) -> tuple[ModelResponseStream, ...]: + if sync_mode: + return tuple(litellm.completion(**kwargs)) + return tuple([chunk async for chunk in await litellm.acompletion(**kwargs)]) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@respx.mock +async def test_anthropic_stream_options_return_usage_after_the_finish_chunk( + monkeypatch: pytest.MonkeyPatch, sync_mode: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + events: Final = ( + { + "type": "message_start", + "message": { + "id": "msg_stream_options", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [], + "usage": { + "input_tokens": 3, + "output_tokens": 1, + "cache_read_input_tokens": 12, + "cache_creation_input_tokens": 4, + }, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "Blue sky."}, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 2}, + }, + {"type": "message_stop"}, + ) + body: Final = "".join( + f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events + ) + route: Final = respx.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response( + 200, text=body, headers={"content-type": "text/event-stream"} + ) + ) + chunks: Final = await _collect_anthropic_stream( + sync_mode, + model="anthropic/claude-sonnet-4-6", + api_key="test-key", + messages=[{"role": "user", "content": "Describe the weather"}], + stream=True, + stream_options={"include_usage": True}, + ) + finish_indices: Final = tuple( + index + for index, chunk in enumerate(chunks) + if chunk.choices and chunk.choices[0].finish_reason is not None + ) + finish_reasons: Final = tuple( + chunk.choices[0].finish_reason + for chunk in chunks + if chunk.choices and chunk.choices[0].finish_reason is not None + ) + usage_indices: Final = tuple( + index + for index, chunk in enumerate(chunks) + if getattr(chunk, "usage", None) is not None + ) + usage_chunk: Final = chunks[usage_indices[0]] + + assert "".join( + chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices + ) == "Blue sky." + assert finish_reasons == ("stop",) + assert finish_indices == (len(chunks) - 2,) + assert usage_indices == (len(chunks) - 1,) + assert usage_chunk.usage.prompt_tokens == 19 + assert usage_chunk.usage.completion_tokens == 2 + assert usage_chunk.usage.total_tokens == 21 + assert usage_chunk.usage.cache_read_input_tokens == 12 + assert usage_chunk.usage.cache_creation_input_tokens == 4 + assert json.loads(route.calls[0].request.content)["stream"] is True + + +_SNOOZE_TOOL: Final = { + "type": "function", + "function": { + "name": "snooze", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, +} + + +def _stream_one_anthropic_tool_use(content_block: dict[str, object]) -> tuple[ChatCompletionDeltaToolCall, ...]: + events: Final = ( + { + "type": "message_start", + "message": { + "id": "msg_snooze", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [], + "usage": {"input_tokens": 12, "output_tokens": 1}, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": content_block}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": ""}}, + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "tool_use", "stop_sequence": None}, + "usage": {"output_tokens": 9}, + }, + {"type": "message_stop"}, + ) + body: Final = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) + with respx.mock: + respx.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response(200, text=body, headers={"content-type": "text/event-stream"}) + ) + chunks: Final = tuple( + litellm.completion( + model="anthropic/claude-sonnet-4-6", + api_key="test-key", + messages=[{"role": "user", "content": "Hit the snooze button."}], + tools=[_SNOOZE_TOOL], + stream=True, + ) + ) + return tuple( + chunk.choices[0].delta.tool_calls[0] for chunk in chunks if chunk.choices and chunk.choices[0].delta.tool_calls + ) + + +def test_anthropic_stream_no_argument_tool_call_streams_an_empty_json_object(): + tool_calls: Final = _stream_one_anthropic_tool_use( + {"type": "tool_use", "id": "toolu_snooze", "name": "snooze", "input": {}} + ) + + assert tuple(tool_call.id for tool_call in tool_calls if tool_call.id is not None) == ("toolu_snooze",) + assert tool_calls[0].function.name == "snooze" + assert json.loads("".join(tool_call.function.arguments or "" for tool_call in tool_calls)) == {} + + +def test_anthropic_stream_tool_call_keeps_the_programmatic_caller(): + caller: Final = {"type": "code_execution_20250825", "tool_id": "srvtoolu_snooze"} + + tool_calls: Final = _stream_one_anthropic_tool_use( + {"type": "tool_use", "id": "toolu_snooze", "name": "snooze", "input": {}, "caller": caller} + ) + + assert tool_calls[0].id == "toolu_snooze" + assert tool_calls[0].model_dump()["caller"] == caller + + def test_streaming_thinking_deltas_count_reasoning_tokens_in_usage(): """Anthropic streaming usage should account for emitted thinking deltas.""" chunks = [ @@ -2367,3 +2650,135 @@ def test_served_model_reaches_assembled_stream_through_custom_stream_wrapper(): assert chunk._hidden_params["provider_response_model"] == served_model assembled: Final = litellm.stream_chunk_builder(chunks=list(chunks), messages=[{"role": "user", "content": "hi"}]) assert assembled._hidden_params["provider_response_model"] == served_model + + +@respx.mock +def test_anthropic_streaming_function_call_arguments_are_assembled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + events: Final = ( + { + "type": "message_start", + "message": { + "id": "msg_tool_stream", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5-20251001", + "content": [], + "usage": {"input_tokens": 5, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "tool_use", + "id": "toolu_weather", + "name": "get_current_weather", + "input": {}, + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"location":"Bos'}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": 'ton","unit":"fahrenheit"}'}, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "tool_use", "stop_sequence": None}, + "usage": {"output_tokens": 7}, + }, + {"type": "message_stop"}, + ) + body: Final = "".join( + f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events + ) + respx.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response( + 200, + text=body, + headers={"content-type": "text/event-stream"}, + ) + ) + messages: Final = [{"role": "user", "content": "Check the weather in Boston"}] + stream: Final = litellm.completion( + model="anthropic/claude-haiku-4-5-20251001", + messages=messages, + tools=[ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + "unit": {"type": "string"}, + }, + "required": ["location", "unit"], + }, + }, + } + ], + stream=True, + api_key="test-anthropic-key", + ) + chunks: Final = tuple(stream) + tool_call_deltas: Final = tuple( + chain.from_iterable( + chunk.choices[0].delta.tool_calls or () for chunk in chunks if chunk.choices + ) + ) + assert tuple( + tool_call.function.name + for tool_call in tool_call_deltas + if tool_call.function.name is not None + ) == ("get_current_weather",) + arguments: Final = "".join( + tool_call.function.arguments or "" for tool_call in tool_call_deltas + ) + assert json.loads(arguments) == {"location": "Boston", "unit": "fahrenheit"} + assembled: Final = litellm.stream_chunk_builder(chunks=list(chunks), messages=messages) + assert assembled.choices[0].message.tool_calls[0].function.name == "get_current_weather" + assert json.loads(assembled.choices[0].message.tool_calls[0].function.arguments) == { + "location": "Boston", + "unit": "fahrenheit", + } + assert assembled.choices[0].finish_reason == "tool_calls" + + +@respx.mock +def test_anthropic_streaming_invalid_key_maps_to_authentication_error() -> None: + route: Final = respx.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response( + 401, + json={ + "type": "error", + "error": { + "type": "authentication_error", + "message": "invalid API key", + }, + }, + ) + ) + + with pytest.raises(litellm.AuthenticationError) as exc_info: + litellm.completion( + model="anthropic/claude-haiku-4-5-20251001", + messages=[{"role": "user", "content": "Say hello"}], + stream=True, + api_key="bad-anthropic-key", + max_retries=0, + ) + + assert exc_info.value.status_code == 401 + assert exc_info.value.llm_provider == "anthropic" + assert route.calls.last.request.headers["x-api-key"] == "bad-anthropic-key" diff --git a/tests/unit/llms/azure/test_azure.py b/tests/unit/llms/azure/test_azure.py index bf1dc14eb48..7907344ae64 100644 --- a/tests/unit/llms/azure/test_azure.py +++ b/tests/unit/llms/azure/test_azure.py @@ -1,12 +1,16 @@ """Tests for litellm/llms/azure/azure.py AzureChatCompletion handler behaviour.""" import asyncio +import json import os import time +from collections.abc import Mapping from typing import Final from unittest.mock import MagicMock, patch +import httpx import pytest +import respx from httpx import Client, Headers from openai import AsyncAzureOpenAI, AzureOpenAI @@ -658,3 +662,134 @@ def test_azure_with_content_safety_error(): ] == "high" ) + + +def _azure_sse_body(chunks: tuple[Mapping[str, object], ...]) -> str: + return "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n" + + +@respx.mock +def test_azure_chat_stream_preserves_text_and_special_characters(): + response_chunks: Final = ( + { + "id": "chatcmpl-azure", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4.1-mini", + "choices": [{"index": 0, "delta": {"content": "café ☃"}, "finish_reason": None}], + }, + { + "id": "chatcmpl-azure", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4.1-mini", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + ) + route: Final = respx.post( + url__regex=r"https://azure-stream\.openai\.azure\.com/.*/chat/completions.*" + ).mock( + return_value=httpx.Response( + 200, + text=_azure_sse_body(response_chunks), + headers={"content-type": "text/event-stream"}, + ) + ) + + stream: Final = litellm.completion( + model="azure/gpt-4.1-mini", + messages=[{"role": "user", "content": "Respond with the tag only"}], + api_base="https://azure-stream.openai.azure.com", + api_version="2024-02-15-preview", + api_key="test-azure-key", + max_retries=0, + stream=True, + ) + chunks: Final = list(stream) + + assert route.call_count == 1 + assert [chunk._hidden_params["custom_llm_provider"] for chunk in chunks] == ["azure"] * len(chunks) + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "café ☃" + assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == ["stop"] + + +@respx.mock +def test_azure_chat_stream_reassembles_tool_call_arguments(): + response_chunks: Final = ( + { + "id": "chatcmpl-azure-tools", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4.1-mini", + "choices": [ + { + "index": 0, + "delta": { + "tool_calls": [ + { + "index": 0, + "id": "call-weather", + "type": "function", + "function": {"name": "get_current_weather", "arguments": '{"city":'}, + } + ] + }, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-azure-tools", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4.1-mini", + "choices": [ + { + "index": 0, + "delta": {"tool_calls": [{"index": 0, "function": {"arguments": '"Boston"}'}}]}, + "finish_reason": "tool_calls", + } + ], + }, + ) + route: Final = respx.post( + url__regex=r"https://azure-tool-stream\.openai\.azure\.com/.*/chat/completions.*" + ).mock( + return_value=httpx.Response( + 200, + text=_azure_sse_body(response_chunks), + headers={"content-type": "text/event-stream"}, + ) + ) + messages: Final = [{"role": "user", "content": "What is the weather in Boston?"}] + stream: Final = litellm.completion( + model="azure/gpt-4.1-mini", + messages=messages, + tools=[ + { + "type": "function", + "function": { + "name": "get_current_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ], + tool_choice="auto", + api_base="https://azure-tool-stream.openai.azure.com", + api_version="2024-02-15-preview", + api_key="test-azure-key", + max_retries=0, + stream=True, + ) + chunks: Final = list(stream) + response: Final = litellm.stream_chunk_builder(chunks, messages=messages) + tool_call: Final = response.choices[0].message.tool_calls[0] + + assert route.call_count == 1 + assert tool_call.function.name == "get_current_weather" + assert json.loads(tool_call.function.arguments) == {"city": "Boston"} + assert response.choices[0].finish_reason == "tool_calls" diff --git a/tests/unit/llms/azure/test_azure_embedding.py b/tests/unit/llms/azure/test_azure_embedding.py index 22ee503ef0d..8d57a3bafce 100644 --- a/tests/unit/llms/azure/test_azure_embedding.py +++ b/tests/unit/llms/azure/test_azure_embedding.py @@ -1,8 +1,13 @@ +import json import os import sys from unittest.mock import MagicMock, patch +import httpx import pytest +import respx + +import litellm sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) @@ -92,3 +97,89 @@ class TestAzureV1AsyncEmbedding: ) assert isinstance(client, AsyncOpenAI) + + +@respx.mock +def test_azure_embedding_forwards_optional_args_in_the_request_body(): + route = respx.post(url__regex=r"https://azure-optional-args\.openai\.azure\.com/.*/embeddings.*").mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], + "model": "text-embedding-ada-002", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + }, + ) + ) + + response = litellm.embedding( + model="azure/text-embedding-ada-002", + input=["hello"], + api_base="https://azure-optional-args.openai.azure.com", + api_version="2023-05-15", + azure_ad_token="test-token", + encoding_format="float", + ) + + request = route.calls.last.request + assert json.loads(request.read()) == { + "model": "text-embedding-ada-002", + "input": ["hello"], + "encoding_format": "float", + "azure_ad_token": "test-token", + } + assert request.headers["authorization"] == "Bearer test-token" + assert response.data[0]["embedding"] == [0.1, 0.2, 0.3] + + +@pytest.mark.asyncio +async def test_aembedding_azure_returns_an_azure_embedding_response( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route = respx_mock.post( + url__regex=r"https://azure-async-embedding\.openai\.azure\.com/.*/embeddings.*" + ).mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.4, 0.5, 0.6]}], + "model": "text-embedding-ada-002", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + }, + ) + ) + + response = await litellm.aembedding( + model="azure/text-embedding-ada-002", + input=["good morning", "another item"], + api_base="https://azure-async-embedding.openai.azure.com", + api_version="2023-05-15", + api_key="test-key", + ) + + assert response._hidden_params["custom_llm_provider"] == "azure" + assert (response.usage.prompt_tokens, response.usage.total_tokens) == (2, 2) + assert response.data[0]["embedding"] == [0.4, 0.5, 0.6] + assert len(route.calls) == 1 + + +def test_azure_embedding_maps_a_scripted_request_timeout(respx_mock: respx.MockRouter) -> None: + route = respx_mock.post( + url__regex=r"https://azure-timeout\.openai\.azure\.com/.*/embeddings.*" + ).mock(side_effect=httpx.ReadTimeout("scripted timeout")) + + with pytest.raises(litellm.Timeout): + litellm.embedding( + model="azure/text-embedding-ada-002", + input=["hello"], + api_base="https://azure-timeout.openai.azure.com", + api_version="2023-05-15", + api_key="test-key", + timeout=0.5, + max_retries=0, + ) + + assert route.call_count == 1 diff --git a/tests/unit/llms/azure/test_azure_exception_mapping.py b/tests/unit/llms/azure/test_azure_exception_mapping.py index 16560c7a1fa..8670a9df2ea 100644 --- a/tests/unit/llms/azure/test_azure_exception_mapping.py +++ b/tests/unit/llms/azure/test_azure_exception_mapping.py @@ -2,6 +2,8 @@ import json from unittest.mock import MagicMock, patch import pytest +import httpx +import respx import litellm @@ -9,6 +11,72 @@ from litellm.exceptions import ContentPolicyViolationError from litellm.litellm_core_utils.exception_mapping_utils import exception_type +@respx.mock +def test_azure_401_maps_to_authentication_error(): + route = respx.post( + url__regex=r"https://azure-bad-key\.openai\.azure\.com/.*/chat/completions.*" + ).mock(return_value=httpx.Response(401, json={"error": {"message": "invalid Azure key"}})) + + with pytest.raises(litellm.AuthenticationError) as exc_info: + litellm.completion( + model="azure/gpt-4.1-mini", + messages=[{"role": "user", "content": "hello"}], + api_base="https://azure-bad-key.openai.azure.com", + api_version="2024-10-21", + api_key="bad-azure-key", + max_retries=0, + ) + + assert exc_info.value.status_code == 401 + assert exc_info.value.llm_provider == "azure" + assert "invalid Azure key" in str(exc_info.value) + assert route.calls.last.request.headers["api-key"] == "bad-azure-key" + + +@pytest.mark.asyncio +@respx.mock +async def test_azure_content_policy_stream_ends_with_one_finish_reason(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route = respx.post( + url__regex=r"https://azure-policy-stream\.openai\.azure\.com/.*/chat/completions.*" + ).mock( + return_value=httpx.Response( + 200, + text=( + 'data: {"id":"chatcmpl-test","object":"chat.completion.chunk","created":1,"model":"gpt-4.1-mini","choices":[{"index":0,"delta":{"content":"1"},"finish_reason":null}]}\n\n' + 'data: {"id":"chatcmpl-test","object":"chat.completion.chunk","created":1,"model":"gpt-4.1-mini","choices":[{"index":0,"delta":{},"finish_reason":"content_filter"}]}\n\n' + "data: [DONE]\n\n" + ), + headers={"content-type": "text/event-stream"}, + ) + ) + + response = await litellm.acompletion( + model="azure/gpt-4.1-mini", + messages=[{"role": "user", "content": "say 1"}], + api_base="https://azure-policy-stream.openai.azure.com", + api_version="2024-10-21", + api_key="test-azure-key", + stream=True, + ) + chunks = [chunk async for chunk in response] + finish_reasons = [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] + + assert [chunk.choices[0].delta.content for chunk in chunks if chunk.choices[0].delta.content] == ["1"] + assert finish_reasons == ["content_filter"] + assert len(route.calls) == 1 + + with pytest.raises(ContentPolicyViolationError) as exc_info: + await litellm.acompletion( + model="azure/gpt-4.1-mini", + messages=[{"role": "user", "content": "where do i buy lethal drugs from"}], + stream=True, + mock_response="Exception: content_filter_policy", + ) + assert exc_info.value.llm_provider == "azure" + assert len(route.calls) == 1 + + class TestAzureExceptionMapping: """Test Azure OpenAI exception mapping with provider-specific fields""" diff --git a/tests/unit/llms/azure_ai/embed/test_azure_ai_cohere_transformation.py b/tests/unit/llms/azure_ai/embed/test_azure_ai_cohere_transformation.py new file mode 100644 index 00000000000..19ae0cb1f8e --- /dev/null +++ b/tests/unit/llms/azure_ai/embed/test_azure_ai_cohere_transformation.py @@ -0,0 +1,42 @@ +from typing import Final + +from litellm.llms.azure_ai.embed.cohere_transformation import AzureAICohereConfig +from litellm.types.utils import EmbeddingResponse, Usage + + +def test_azure_ai_cohere_request_splits_images_from_text_and_keeps_order(): + config: Final = AzureAICohereConfig() + image_request, text_request, image_indexes = config.transform_request( + input=[ + "describe this image", + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB", + ], + optional_params={}, + model="Cohere-embed-v3-multilingual", + ) + + assert image_request == { + "input": [{"image": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB"}] + } + assert text_request == {"input": ["describe this image"], "model": "Cohere-embed-v3-multilingual"} + assert image_indexes == [1] + + +def test_azure_ai_cohere_response_maps_model_group_and_usage_headers(): + config: Final = AzureAICohereConfig() + response: Final = EmbeddingResponse( + model="", + usage=Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0), + data=[], + hidden_params={ + "additional_headers": { + "llm_provider-num_tokens": "7", + "llm_provider-azureml-model-group": "offer-cohere-embed-multili-paygo", + } + }, + ) + + result: Final = config.transform_response(response) + + assert result.model == "Cohere-embed-v3-multilingual" + assert result.usage.prompt_tokens == 7 diff --git a/tests/unit/llms/azure_ai/embed/test_azure_ai_embed_handler.py b/tests/unit/llms/azure_ai/embed/test_azure_ai_embed_handler.py index 0401629171d..2b8b0a462bf 100644 --- a/tests/unit/llms/azure_ai/embed/test_azure_ai_embed_handler.py +++ b/tests/unit/llms/azure_ai/embed/test_azure_ai_embed_handler.py @@ -1,3 +1,5 @@ +import json + import httpx import pytest import respx @@ -64,6 +66,31 @@ def test_azure_ai_embedding_calls_foundry_models_route(): api_key="fake-key", ) - assert route.called - assert response.data is not None - assert len(response.data) == 1 + assert route.call_count == 1 + assert [item["embedding"] for item in response.data] == [[0.1, 0.2]] + + +@respx.mock +def test_azure_ai_cohere_image_embedding_calls_image_route(): + route = respx.post("https://my-foundry.services.ai.azure.com/models/images/embeddings").mock( + return_value=httpx.Response( + 200, + json={ + "data": [{"object": "embedding", "embedding": [0.1, 0.2], "index": 0}], + "model": "Cohere-embed-v3-multilingual", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + }, + ) + ) + + response = embedding( + model="azure_ai/Cohere-embed-v3-multilingual", + input=["data:image/png;base64,aGVsbG8="], + api_base="https://my-foundry.services.ai.azure.com", + api_key="fake-key", + ) + + assert json.loads(route.calls.last.request.read()) == {"input": [{"image": "data:image/png;base64,aGVsbG8="}]} + assert set(dict(response).keys()) == {"object", "data", "model", "usage"} + assert response.data[0]["embedding"] == [0.1, 0.2] + assert (response.usage.prompt_tokens, response.usage.total_tokens) == (2, 2) diff --git a/tests/unit/llms/bedrock/chat/test_invoke_handler.py b/tests/unit/llms/bedrock/chat/test_invoke_handler.py index d5fe5d8cf65..fbb4980df24 100644 --- a/tests/unit/llms/bedrock/chat/test_invoke_handler.py +++ b/tests/unit/llms/bedrock/chat/test_invoke_handler.py @@ -1052,6 +1052,180 @@ async def test_async_converse_stream_with_an_empty_200_body_raises_instead_of_an _assert_empty_stream_surfaced_as_bad_gateway(exc_info.value) +def _assert_converse_stream_text_and_finish_reason(chunks: Sequence[ModelResponseStream]) -> None: + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "Streamed reply" + assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == ["stop"] + + +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.asyncio +async def test_converse_streaming_completion_yields_text_and_finish_reason( + _aws_test_credentials: None, + async_mode: bool, +) -> None: + event_stream_body: Final = b"".join( + ( + _converse_event_frame("messageStart", {"role": "assistant"}), + _converse_event_frame( + "contentBlockDelta", + {"contentBlockIndex": 0, "delta": {"text": "Streamed "}}, + ), + _converse_event_frame( + "contentBlockDelta", + {"contentBlockIndex": 0, "delta": {"text": "reply"}}, + ), + _converse_event_frame("contentBlockStop", {"contentBlockIndex": 0}), + _converse_event_frame("messageStop", {"stopReason": "end_turn"}), + _converse_event_frame( + "metadata", + { + "usage": {"inputTokens": 3, "outputTokens": 2, "totalTokens": 5}, + "metrics": {"latencyMs": 1}, + }, + ), + ) + ) + def handle_request(request: httpx.Request) -> httpx.Response: + assert request.url.path.endswith("/converse-stream") + return httpx.Response(200, content=event_stream_body, headers=_event_stream_headers()) + + transport: Final = httpx.MockTransport(handle_request) + if async_mode: + async_client: Final = AsyncHTTPHandler(transport=transport) + async_stream: Final = await litellm.acompletion( + model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=[{"role": "user", "content": "Say hello"}], + stream=True, + client=async_client, + ) + async_chunks: Final = tuple([chunk async for chunk in async_stream]) + _assert_converse_stream_text_and_finish_reason(async_chunks) + else: + sync_client: Final = HTTPHandler(client=httpx.Client(transport=transport)) + sync_stream: Final = litellm.completion( + model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=[{"role": "user", "content": "Say hello"}], + stream=True, + client=sync_client, + ) + sync_chunks: Final = tuple(sync_stream) + _assert_converse_stream_text_and_finish_reason(sync_chunks) + + +def test_mistral_invoke_stream_yields_text_and_finish_reason( + _aws_test_credentials: None, +) -> None: + event_stream_body: Final = b"".join( + _bedrock_event_stream_frame(chunk) + for chunk in ( + {"outputs": [{"text": "A court case moves ", "stop_reason": None}]}, + {"outputs": [{"text": "through the courts.", "stop_reason": "stop"}]}, + ) + ) + + def handle_request(request: httpx.Request) -> httpx.Response: + assert request.url.path.endswith("/invoke-with-response-stream") + return httpx.Response( + 200, + content=event_stream_body, + headers=_event_stream_headers(), + ) + + transport: Final = httpx.MockTransport(handle_request) + client: Final = HTTPHandler(client=httpx.Client(transport=transport)) + stream: Final = litellm.completion( + model="bedrock/mistral.mixtral-8x7b-instruct-v0:1", + messages=[{"role": "user", "content": "Where does a court case go?"}], + stream=True, + client=client, + ) + chunks: Final = tuple(stream) + + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == ( + "A court case moves through the courts." + ) + assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == [ + "stop" + ] + + +def test_converse_streaming_tool_call_arguments_are_valid_json( + _aws_test_credentials: None, +) -> None: + event_stream_body: Final = b"".join( + _converse_event_frame(event_type, body) + for event_type, body in ( + ("messageStart", {"role": "assistant"}), + ( + "contentBlockStart", + { + "contentBlockIndex": 0, + "start": { + "toolUse": { + "toolUseId": "toolu_weather", + "name": "get_current_weather", + } + }, + }, + ), + ( + "contentBlockDelta", + {"contentBlockIndex": 0, "delta": {"toolUse": {"input": '{"location":"Bos'}}}, + ), + ( + "contentBlockDelta", + {"contentBlockIndex": 0, "delta": {"toolUse": {"input": 'ton","unit":"fahrenheit"}'}}}, + ), + ("contentBlockStop", {"contentBlockIndex": 0}), + ("messageStop", {"stopReason": "tool_use"}), + ("metadata", {"usage": {"inputTokens": 4, "outputTokens": 7}}), + ) + ) + + def handle_request(request: httpx.Request) -> httpx.Response: + assert request.url.path.endswith("/converse-stream") + return httpx.Response( + 200, + content=event_stream_body, + headers=_event_stream_headers(), + ) + + transport: Final = httpx.MockTransport(handle_request) + client: Final = HTTPHandler(client=httpx.Client(transport=transport)) + messages: Final = [{"role": "user", "content": "What is the weather in Boston?"}] + stream: Final = litellm.completion( + model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=messages, + tools=[ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + "unit": {"type": "string"}, + }, + "required": ["location", "unit"], + }, + }, + } + ], + stream=True, + client=client, + ) + chunks: Final = tuple(stream) + assembled: Final = litellm.stream_chunk_builder(chunks=list(chunks), messages=messages) + assert assembled.choices[0].message.tool_calls[0].function.name == "get_current_weather" + assert json.loads(assembled.choices[0].message.tool_calls[0].function.arguments) == { + "location": "Boston", + "unit": "fahrenheit", + } + assert assembled.choices[0].finish_reason == "tool_calls" + + _UPSTREAM_REJECTION: Final = "structured output schema uses unsupported regex negative look-ahead" _CUSTOMER_REJECTION_EVENT_TYPE: Final = "validationException" diff --git a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py index 01bfa29f71c..d755b8e15f9 100644 --- a/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/unit/llms/bedrock/embed/test_bedrock_embedding.py @@ -6,8 +6,10 @@ from unittest.mock import AsyncMock, MagicMock import pytest import httpx +import respx import litellm +from litellm.caching.caching import Cache from litellm.llms.bedrock.embed.twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.bedrock.embed.embedding import BedrockEmbedding @@ -127,6 +129,65 @@ def test_bedrock_embedding_with_env_variable_bearer_token( assert headers["Authorization"] == f"Bearer {test_api_key}" +@pytest.mark.parametrize( + "model", + ["amazon.titan-embed-text-v1", "amazon.titan-embed-image-v1", "amazon.titan-embed-text-v2:0"], +) +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_bedrock_titan_embedding_cache_hit_avoids_a_second_request( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, model: str, sync_mode: bool +) -> None: + monkeypatch.setattr(litellm, "cache", Cache()) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route = respx_mock.post(f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model}/invoke").mock( + return_value=httpx.Response(200, json=titan_embedding_response) + ) + request = { + "model": f"bedrock/{model}", + "input": test_input, + "api_key": "test-bearer-token", + "aws_region_name": "us-east-1", + "aws_bedrock_runtime_endpoint": "https://bedrock-runtime.us-east-1.amazonaws.com", + "caching": True, + } + + if sync_mode: + response_1 = litellm.embedding(**request) + response_2 = litellm.embedding(**request) + else: + response_1 = await litellm.aembedding(**request) + response_2 = await litellm.aembedding(**request) + + assert json.loads(route.calls.last.request.read()) == {"inputText": test_input} + assert response_1.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert response_1.usage.prompt_tokens == 10 + assert response_2._hidden_params["cache_hit"] is True + assert response_2.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert route.call_count == 1 + + +def test_bedrock_cohere_embedding_returns_the_scripted_vector(respx_mock: respx.MockRouter) -> None: + route = respx_mock.post( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/cohere.embed-multilingual-v3/invoke" + ).mock( + return_value=httpx.Response( + 200, + json={"embeddings": [[0.3, 0.2, 0.1]], "inputTextTokenCount": 4}, + ) + ) + + response = litellm.embedding( + model="bedrock/cohere.embed-multilingual-v3", + input=["hello"], + api_key="test-bearer-token", + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint="https://bedrock-runtime.us-east-1.amazonaws.com", + ) + + assert json.loads(route.calls.last.request.read()) == {"input_type": "search_document", "texts": ["hello"]} + assert response.data == [{"object": "embedding", "index": 0, "embedding": [0.3, 0.2, 0.1]}] + + @pytest.mark.asyncio async def test_async_bedrock_embedding_with_bearer_token(): """Test async embedding functionality with bearer token authentication""" diff --git a/tests/unit/llms/cohere/chat/test_cohere_transformation.py b/tests/unit/llms/cohere/chat/test_cohere_transformation.py index c070d410755..992266c5e1d 100644 --- a/tests/unit/llms/cohere/chat/test_cohere_transformation.py +++ b/tests/unit/llms/cohere/chat/test_cohere_transformation.py @@ -264,3 +264,25 @@ async def test_cohere_documents_options_in_request_body(): # Validate that documents and citation_options are in the request body assert "documents" in request_data assert request_data["documents"] == test_documents + + +def test_cohere_invalid_max_tokens_maps_to_bad_request_error( + respx_mock: respx.MockRouter, _cohere_httpx_transport: None +) -> None: + route: Final = respx_mock.post(COHERE_V2_CHAT_URL).respond( + 400, json={"message": "invalid request: max_tokens must be a number"} + ) + + with pytest.raises(litellm.BadRequestError) as error: + litellm.completion( + model="command-nightly", + messages=[{"content": "hey, how's it going?", "role": "user"}], + max_tokens="hello world", + api_key="cohere-test-key", + ) + + assert error.value.status_code == 400 + assert error.value.llm_provider == "cohere" + assert "CohereException" in str(error.value) + assert "max_tokens must be a number" in str(error.value) + assert json.loads(route.calls.last.request.content)["max_tokens"] == "hello world" diff --git a/tests/unit/llms/cohere/embed/test_cohere_embedding_transformation.py b/tests/unit/llms/cohere/embed/test_cohere_embedding_transformation.py new file mode 100644 index 00000000000..516ea1f5221 --- /dev/null +++ b/tests/unit/llms/cohere/embed/test_cohere_embedding_transformation.py @@ -0,0 +1,179 @@ +from datetime import datetime, timezone +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.cohere.embed.transformation import CohereEmbeddingConfig +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.types.utils import CallTypes, EmbeddingResponse + + +def _logging_obj(inputs: list[str]) -> Logging: + logging_obj: Final = Logging( + model="embed-english-v3.0", + messages=inputs, + stream=False, + call_type=CallTypes.embedding, + start_time=datetime(2025, 1, 1, tzinfo=timezone.utc), + litellm_call_id="test-call", + function_id="test-function", + ) + logging_obj.model_call_details = {"input": inputs} + return logging_obj + + +def test_cohere_text_embedding_request_maps_openai_params(): + config: Final = CohereEmbeddingConfig() + request: Final = config.transform_embedding_request( + model="embed-english-v3.0", + input=["hello", "world"], + optional_params={"output_dimension": 256, "embedding_types": ["float"]}, + headers={}, + ) + + assert request == { + "model": "embed-english-v3.0", + "texts": ["hello", "world"], + "input_type": litellm.COHERE_DEFAULT_EMBEDDING_INPUT_TYPE, + "output_dimension": 256, + "embedding_types": ["float"], + } + + +def test_cohere_image_embedding_request_uses_images(): + config: Final = CohereEmbeddingConfig() + request: Final = config.transform_embedding_request( + model="embed-english-v3.0", + input=["data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB"], + optional_params={}, + headers={}, + ) + + assert request == { + "model": "embed-english-v3.0", + "images": ["data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAAB"], + "input_type": "image", + } + + +def test_cohere_embedding_response_preserves_vectors_and_provider_usage(): + config: Final = CohereEmbeddingConfig() + inputs: Final = ["hello", "world"] + logging_obj: Final = _logging_obj(inputs) + response: Final = config.transform_embedding_response( + model="embed-english-v3.0", + raw_response=httpx.Response( + 200, + json={ + "embeddings": {"float": [[0.1, 0.2], [0.3, 0.4]]}, + "meta": {"billed_units": {"input_tokens": 5}}, + }, + ), + model_response=EmbeddingResponse(), + logging_obj=logging_obj, + api_key="test-key", + request_data={"texts": inputs}, + optional_params={}, + litellm_params={}, + ) + + assert response.model == "embed-english-v3.0" + assert response.data == [ + {"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}, + {"object": "embedding", "index": 1, "embedding": [0.3, 0.4]}, + ] + assert (response.usage.prompt_tokens, response.usage.total_tokens) == (5, 5) + + +def test_cohere_embedding_response_counts_tokens_when_provider_usage_is_missing(): + config: Final = CohereEmbeddingConfig() + inputs: Final = ["hello"] + logging_obj: Final = _logging_obj(inputs) + response: Final = config.transform_embedding_response( + model="embed-english-v3.0", + raw_response=httpx.Response(200, json={"embeddings": {"float": [[0.1]]}}), + model_response=EmbeddingResponse(), + logging_obj=logging_obj, + api_key="test-key", + request_data={"texts": inputs}, + optional_params={}, + litellm_params={}, + ) + + assert response.usage.prompt_tokens == len(litellm.encoding.encode("hello")) + assert response.usage.total_tokens == response.usage.prompt_tokens + + +def test_cohere_embedding_usage_includes_text_and_image_token_details(): + config: Final = CohereEmbeddingConfig() + usage: Final = config._calculate_usage( + input=["hello"], + encoding=litellm.encoding, + meta={"billed_units": {"input_tokens": 5, "images": 2}}, + ) + + assert usage.prompt_tokens == 7 + assert (usage.prompt_tokens_details.text_tokens, usage.prompt_tokens_details.image_tokens) == (5, 2) + + +def test_cohere_embedding_error_class_preserves_status_and_message(): + error: Final = CohereEmbeddingConfig().get_error_class( + error_message="invalid embedding request", + status_code=400, + headers={}, + ) + + assert error.status_code == 400 + assert "invalid embedding request" in str(error) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_embedding_with_extra_headers_are_sent_to_cohere( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, sync_mode: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.cohere.ai/v2/embed").mock( + return_value=httpx.Response( + 200, + json={"embeddings": {"float": [[0.1, 0.2]]}, "meta": {"billed_units": {"input_tokens": 1}}}, + ) + ) + request: Final = { + "model": "cohere/embed-english-v3.0", + "input": ["hello"], + "api_key": "test-key", + "extra_headers": {"x-request-id": "test-request"}, + } + + response: Final = litellm.embedding(**request) if sync_mode else await litellm.aembedding(**request) + + assert route.calls.last.request.headers["x-request-id"] == "test-request" + assert response.data[0]["embedding"] == [0.1, 0.2] + + +async def test_aembedding_sends_extra_headers_through_the_caller_supplied_async_client() -> None: + requests: Final[list[httpx.Request]] = [] + + def handle_request(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response( + 200, + json={"embeddings": {"float": [[0.3, 0.4]]}, "meta": {"billed_units": {"input_tokens": 2}}}, + ) + + response: Final = await litellm.aembedding( + model="cohere/embed-english-v3.0", + input=["hello world"], + api_key="test-key", + extra_headers={"my-test-param": "hello-world"}, + client=AsyncHTTPHandler(transport=httpx.MockTransport(handle_request)), + ) + + assert len(requests) == 1 + assert str(requests[0].url) == "https://api.cohere.ai/v2/embed" + assert requests[0].headers["my-test-param"] == "hello-world" + assert response.data[0]["embedding"] == [0.3, 0.4] diff --git a/tests/unit/llms/custom_httpx/test_http_handler.py b/tests/unit/llms/custom_httpx/test_http_handler.py index b6b0db69162..b9dbf7f9359 100644 --- a/tests/unit/llms/custom_httpx/test_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_http_handler.py @@ -14,9 +14,11 @@ from unittest.mock import MagicMock, patch import certifi import httpx import pytest +import respx from aiohttp import ClientSession, TCPConnector import litellm +from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.custom_httpx.aiohttp_transport import LiteLLMAiohttpTransport from litellm.llms.custom_httpx.http_handler import ( _CLIENT_REFCOUNT_WHEN_HANDLER_IS_SOLE_REFERRER, @@ -2051,3 +2053,73 @@ def test_realtime_ssl_for_url_turns_ssl_verify_false_into_an_unverified_tls_cont assert isinstance(selected, ssl.SSLContext) assert selected.verify_mode == ssl.CERT_NONE assert selected.check_hostname is False + + +def _ollama_ssl_messages() -> list[dict[str, str]]: + return [{"role": "user", "content": "What's the weather like in San Francisco?"}] + + +def _ollama_generate_route(respx_mock: respx.MockRouter, stream: bool) -> respx.Route: + final: Final = {"model": "llama3.1", "response": "", "done": True, "prompt_eval_count": 2, "eval_count": 1} + if stream: + return respx_mock.post("http://ollama.test:11434/api/generate").respond( + text=json.dumps({"model": "llama3.1", "response": "Sunny", "done": False}) + "\n" + json.dumps(final) + "\n", + headers={"content-type": "application/x-ndjson"}, + ) + return respx_mock.post("http://ollama.test:11434/api/generate").respond(json={**final, "response": "Sunny"}) + + +def _cached_clients_verify_modes( + cache: LLMClientCache, client_type: type[HTTPHandler | AsyncHTTPHandler] +) -> tuple[ssl.VerifyMode, ...]: + return tuple( + client.client._transport._pool._ssl_context.verify_mode + for client in cache.cache_dict.values() + if isinstance(client, client_type) + ) + + +def test_ollama_ssl_verify_false_builds_a_client_that_skips_certificate_checks( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + client_cache: Final = LLMClientCache() + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", client_cache) + route: Final = _ollama_generate_route(respx_mock, stream=False) + + response: Final = litellm.completion( + model="ollama/llama3.1", + messages=_ollama_ssl_messages(), + api_base="http://ollama.test:11434", + ssl_verify=False, + ) + + assert route.call_count == 1 + assert response.choices[0].message.content == "Sunny" + assert _cached_clients_verify_modes(client_cache, HTTPHandler) == (ssl.CERT_NONE,) + + +@pytest.mark.parametrize("stream", [True, False]) +async def test_async_ollama_ssl_verify_false_builds_a_client_that_skips_certificate_checks( + stream: bool, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + client_cache: Final = LLMClientCache() + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", client_cache) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = _ollama_generate_route(respx_mock, stream=stream) + + response: Final = await litellm.acompletion( + model="ollama/llama3.1", + messages=_ollama_ssl_messages(), + api_base="http://ollama.test:11434", + ssl_verify=False, + stream=stream, + ) + text: Final = ( + "".join([chunk.choices[0].delta.content or "" async for chunk in response]) + if stream + else response.choices[0].message.content + ) + + assert route.call_count == 1 + assert text == "Sunny" + assert _cached_clients_verify_modes(client_cache, AsyncHTTPHandler) == (ssl.CERT_NONE,) diff --git a/tests/unit/llms/fireworks_ai/embed/__init__.py b/tests/unit/llms/fireworks_ai/embed/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/fireworks_ai/embed/test_fireworks_ai_embedding_transformation.py b/tests/unit/llms/fireworks_ai/embed/test_fireworks_ai_embedding_transformation.py new file mode 100644 index 00000000000..b46ed5c7aef --- /dev/null +++ b/tests/unit/llms/fireworks_ai/embed/test_fireworks_ai_embedding_transformation.py @@ -0,0 +1,68 @@ +import json +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.llms.fireworks_ai.embed.fireworks_ai_transformation import FireworksAIEmbeddingConfig + + +def test_fireworks_nomic_embedding_supports_and_maps_dimensions(): + config: Final = FireworksAIEmbeddingConfig() + params: Final = config.map_openai_params( + non_default_params={"dimensions": 256, "encoding_format": "float"}, + optional_params={}, + model="nomic-ai/nomic-embed-text-v1.5", + ) + + assert config.get_supported_openai_params("nomic-ai/nomic-embed-text-v1.5") == ["dimensions"] + assert params == {"dimensions": 256} + assert config.is_fireworks_embedding_model("nomic-ai/nomic-embed-text-v1.5") + + +def test_fireworks_non_nomic_embedding_drops_dimensions(): + config: Final = FireworksAIEmbeddingConfig() + + assert config.get_supported_openai_params("other/model") == [] + assert config.map_openai_params( + non_default_params={"dimensions": 256}, + optional_params={}, + model="other/model", + ) == {} + assert not config.is_fireworks_embedding_model("other/model") + + +@respx.mock +def test_fireworks_embeddings_forwards_supported_dimensions(monkeypatch: pytest.MonkeyPatch): + model_key: Final = "fireworks_ai/nomic-ai/nomic-embed-text-v1.5" + monkeypatch.setitem(litellm.model_cost, model_key, {**litellm.model_cost[model_key], "input_cost_per_token": 2.5e-08}) + route: Final = respx.post("https://fireworks.example/v1/embeddings").mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "nomic-ai/nomic-embed-text-v1.5", + "usage": {"prompt_tokens": 4, "total_tokens": 4}, + }, + ) + ) + + response: Final = litellm.embedding( + model="fireworks_ai/nomic-ai/nomic-embed-text-v1.5", + input=["hello"], + api_base="https://fireworks.example/v1", + api_key="test-key", + dimensions=2, + ) + + assert json.loads(route.calls.last.request.content) == { + "input": ["hello"], + "model": "nomic-ai/nomic-embed-text-v1.5", + "dimensions": 2, + } + assert response.data[0]["embedding"] == [0.1, 0.2] + assert response.usage.prompt_tokens == 4 + assert response._hidden_params["response_cost"] == pytest.approx(4 * 2.5e-08) diff --git a/tests/unit/llms/fireworks_ai/test_fireworks_ai_text_completion.py b/tests/unit/llms/fireworks_ai/test_fireworks_ai_text_completion.py new file mode 100644 index 00000000000..de9af9979ce --- /dev/null +++ b/tests/unit/llms/fireworks_ai/test_fireworks_ai_text_completion.py @@ -0,0 +1,68 @@ +import json +from typing import Final + +import httpx +import respx + +from litellm import text_completion + +FIREWORKS_COMPLETIONS_URL: Final = "https://api.fireworks.ai/inference/v1/completions" +FIREWORKS_MODEL: Final = "accounts/fireworks/models/llama-v3p1-8b-instruct" + + +def _fireworks_completion_response() -> dict[str, object]: + return { + "id": "cmpl-fireworks-migration", + "object": "text_completion", + "created": 1, + "model": FIREWORKS_MODEL, + "choices": [ + { + "text": "hello world", + "index": 0, + "logprobs": {"tokens": ["hello", " world"], "token_logprobs": [-0.1, -0.2]}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + } + + +def test_fireworks_text_completion_sends_prompt_array(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(FIREWORKS_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, json=_fireworks_completion_response()) + ) + + response: Final = text_completion( + model=f"fireworks_ai/{FIREWORKS_MODEL}", + prompt=["What's the weather in SF?", "How is Manchester?"], + max_tokens=5, + api_key="fireworks-test-key", + ) + + request_body: Final = json.loads(route.calls.last.request.read()) + assert request_body["prompt"] == ["What's the weather in SF?", "How is Manchester?"] + assert [choice.text for choice in response.choices] == ["hello world"] + assert route.calls.last.request.headers["Authorization"] == "Bearer fireworks-test-key" + + +def test_fireworks_text_completion_echo_returns_prompt_and_logprobs(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(FIREWORKS_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, json=_fireworks_completion_response()) + ) + + response: Final = text_completion( + model=f"fireworks_ai/{FIREWORKS_MODEL}", + prompt="hello", + max_tokens=1, + stop="\n", + logprobs=1, + echo=True, + api_key="fireworks-test-key", + ) + + request_body: Final = json.loads(route.calls.last.request.read()) + assert request_body["echo"] is True + assert request_body["stop"] == "\n" + assert response.choices[0].text == "hello world" + assert response.choices[0].logprobs.token_logprobs == [-0.1, -0.2] diff --git a/tests/unit/llms/gemini/test_gemini_streaming.py b/tests/unit/llms/gemini/test_gemini_streaming.py new file mode 100644 index 00000000000..b9ad2018738 --- /dev/null +++ b/tests/unit/llms/gemini/test_gemini_streaming.py @@ -0,0 +1,219 @@ +import json +from collections.abc import Mapping +from typing import Final + +import httpx +import pytest +import respx + +import litellm + + +def _sse_body(chunks: tuple[Mapping[str, object], ...]) -> str: + return "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n" + + +@pytest.mark.asyncio +@respx.mock +async def test_async_gemini_streaming_tool_call_arguments_are_valid( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + chunks: Final = ( + { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "generate_series_of_questions", + "args": { + "questions": [ + "What is a bridge?", + "How is it designed?", + "How is it built?", + ] + }, + } + } + ], + "role": "model", + }, + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 5, + "candidatesTokenCount": 7, + "totalTokenCount": 12, + }, + }, + { + "candidates": [ + { + "content": {"parts": [], "role": "model"}, + "finishReason": "STOP", + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 5, + "candidatesTokenCount": 7, + "totalTokenCount": 12, + }, + }, + ) + route: Final = respx.post( + url__regex=r"https://generativelanguage\.googleapis\.com/v1beta/models/gemini-2\.5-flash-lite:streamGenerateContent.*" + ).mock( + return_value=httpx.Response( + 200, + text=_sse_body(chunks), + headers={"content-type": "text/event-stream"}, + ) + ) + + stream: Final = await litellm.acompletion( + model="gemini/gemini-2.5-flash-lite", + messages=[ + {"role": "system", "content": "You are an AI assistant"}, + {"role": "user", "content": "Generate three questions about civil engineering."}, + ], + tools=[ + { + "type": "function", + "function": { + "name": "generate_series_of_questions", + "description": "Generate questions", + "parameters": { + "type": "object", + "properties": { + "questions": { + "type": "array", + "items": {"type": "string"}, + } + }, + "required": ["questions"], + }, + }, + } + ], + stream=True, + api_key="test-gemini-key", + ) + response_chunks: Final = [chunk async for chunk in stream] + tool_calls: Final = [ + tool_call + for chunk in response_chunks + for tool_call in chunk.choices[0].delta.tool_calls or () + ] + + assert route.call_count == 1 + assert response_chunks[0].choices[0].delta.role == "assistant" + assert tuple(chunk.choices[0].finish_reason for chunk in response_chunks) == (None, "tool_calls") + assert len(tool_calls) == 1 + assert tool_calls[0].function.name == "generate_series_of_questions" + assert json.loads(tool_calls[0].function.arguments) == { + "questions": [ + "What is a bridge?", + "How is it designed?", + "How is it built?", + ] + } + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +@respx.mock +async def test_gemini_streaming_legacy_function_call_arguments_are_valid( + sync_mode: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + chunks: Final = ( + { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "get_current_weather", + "args": {"location": "Boston", "unit": "fahrenheit"}, + } + } + ], + "role": "model", + }, + "index": 0, + } + ] + }, + { + "candidates": [ + { + "content": {"parts": [], "role": "model"}, + "finishReason": "STOP", + "index": 0, + } + ] + }, + ) + respx.post( + url__regex=r"https://generativelanguage\.googleapis\.com/v1beta/models/gemini-2\.5-flash-lite:streamGenerateContent.*" + ).mock( + return_value=httpx.Response( + 200, + text=_sse_body(chunks), + headers={"content-type": "text/event-stream"}, + ) + ) + messages: Final = [{"role": "user", "content": "What is the weather in Boston?"}] + functions: Final = [ + { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + "unit": {"type": "string"}, + }, + "required": ["location", "unit"], + }, + } + ] + if sync_mode: + stream_chunks: Final = list( + litellm.completion( + model="gemini/gemini-2.5-flash-lite", + messages=messages, + functions=functions, + stream=True, + api_key="test-gemini-key", + ) + ) + else: + stream: Final = await litellm.acompletion( + model="gemini/gemini-2.5-flash-lite", + messages=messages, + functions=functions, + stream=True, + api_key="test-gemini-key", + ) + stream_chunks: Final = [chunk async for chunk in stream] + + function_calls: Final = tuple( + chunk.choices[0].delta.function_call + for chunk in stream_chunks + if chunk.choices and chunk.choices[0].delta.function_call is not None + ) + assert tuple( + function_call.name for function_call in function_calls if function_call.name is not None + ) == ("get_current_weather",) + arguments: Final = "".join(function_call.arguments or "" for function_call in function_calls) + assert json.loads(arguments) == {"location": "Boston", "unit": "fahrenheit"} + rebuilt: Final = litellm.stream_chunk_builder(chunks=list(stream_chunks), messages=messages) + assert rebuilt.choices[0].message.function_call.name == "get_current_weather" + assert json.loads(rebuilt.choices[0].message.function_call.arguments) == {"location": "Boston", "unit": "fahrenheit"} diff --git a/tests/unit/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_transformation.py b/tests/unit/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_transformation.py index 5854b1596b4..e0a34a2c571 100644 --- a/tests/unit/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_transformation.py +++ b/tests/unit/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_transformation.py @@ -6,9 +6,11 @@ especially ensuring that encoding_format is not included when not provided. """ import json +from typing import Final from unittest.mock import Mock, patch import pytest +import respx import litellm @@ -326,3 +328,23 @@ class TestHostedVLLMEmbeddingTransformation: if __name__ == "__main__": pytest.main([__file__, "-v", "-s"]) + + +def test_hosted_vllm_embedding_posts_to_the_env_api_base( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://hosted-vllm.test") + route: Final = respx_mock.post("http://hosted-vllm.test/embeddings").respond( + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.5, 0.25]}], + "model": "jina-embeddings-v3", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + } + ) + + response: Final = litellm.embedding(model="hosted_vllm/jina-embeddings-v3", input=["Hello world"]) + + assert route.call_count == 1 + assert json.loads(route.calls.last.request.content) == {"model": "jina-embeddings-v3", "input": ["Hello world"]} + assert response.data[0]["embedding"] == [0.5, 0.25] diff --git a/tests/unit/llms/huggingface/embedding/test_huggingface_embedding_handler.py b/tests/unit/llms/huggingface/embedding/test_huggingface_embedding_handler.py index 0384fb796d9..ea1e04291ca 100644 --- a/tests/unit/llms/huggingface/embedding/test_huggingface_embedding_handler.py +++ b/tests/unit/llms/huggingface/embedding/test_huggingface_embedding_handler.py @@ -1,4 +1,5 @@ import json +from typing import Final from unittest.mock import patch, MagicMock, AsyncMock @@ -154,3 +155,45 @@ class TestHuggingFaceEmbedding: assert "sentences" in request_data["inputs"] assert request_data["inputs"]["source_sentence"] == input_text[0] assert request_data["inputs"]["sentences"] == input_text[1:] + + def test_hf_sentence_similarity_transform_keeps_provider_options(self): + from litellm.llms.huggingface.embedding.handler import HuggingFaceEmbedding + + request_data: Final = HuggingFaceEmbedding()._transform_input( + input=["source sentence", "candidate sentence"], + model="BAAI/bge-m3", + call_type="sync", + optional_params={"input_type": "sentence-similarity", "min_length": 2}, + embed_url="https://huggingface.example/embeddings", + ) + + assert request_data == { + "inputs": {"source_sentence": "source sentence", "sentences": ["candidate sentence"]}, + "parameters": {"min_length": 2}, + } + + +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_hf_embedding_sentence_sim_sends_one_similarity_request( + sync_mode: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("HUGGINGFACE_API_KEY", "hf-test-token") + with respx.mock(assert_all_called=False) as hf: + hf.get(url__regex=r"https://huggingface\.co/api/models/.*").respond(json={"pipeline_tag": "sentence-similarity"}) + hf.get(url__regex=r"https://huggingface\.co/.*/config\.json").respond(json={"max_position_embeddings": 512}) + inference: Final = hf.post( + "https://router.huggingface.co/hf-inference/pipeline/sentence-similarity/sentence-transformers/TaylorAI/bge-micro-v2" + ).respond(json=[0.7708950042724609]) + request: Final = { + "model": "huggingface/sentence-transformers/TaylorAI/bge-micro-v2", + "input": ["good morning from litellm", "this is another item"], + } + response: Final = litellm.embedding(**request) if sync_mode else await litellm.aembedding(**request) + + assert inference.call_count == 1 + assert json.loads(inference.calls.last.request.content) == { + "inputs": {"source_sentence": "good morning from litellm", "sentences": ["this is another item"]} + } + assert inference.calls.last.request.headers["Authorization"] == "Bearer hf-test-token" + assert response.data == [{"object": "embedding", "index": 0, "embedding": 0.7708950042724609}] diff --git a/tests/unit/llms/jina_ai/embedding/test_jina_embedding_transformation.py b/tests/unit/llms/jina_ai/embedding/test_jina_embedding_transformation.py index 579ba2d75ef..1e759ab4afd 100644 --- a/tests/unit/llms/jina_ai/embedding/test_jina_embedding_transformation.py +++ b/tests/unit/llms/jina_ai/embedding/test_jina_embedding_transformation.py @@ -1,9 +1,14 @@ +import json +from typing import Final from unittest.mock import MagicMock import httpx import pytest +import respx from pydantic import ValidationError +import litellm + from litellm.llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig from litellm.types.utils import EmbeddingResponse @@ -192,3 +197,40 @@ def test_transform_embedding_response_invalid_field_is_reported_by_name(field: s assert exc_info.value.title == "EmbeddingResponse" assert [error["loc"] for error in exc_info.value.errors()] == [(field,)] + + +_PIXEL_PNG: Final = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" + + +@pytest.mark.parametrize( + ("input_data", "expected_payload_input"), + [ + (["hello world", "foo bar"], ["hello world", "foo bar"]), + ( + ["A picture of a cat", f"data:image/png;base64,{_PIXEL_PNG}"], + [{"text": "A picture of a cat"}, {"image": _PIXEL_PNG}], + ), + ([f"data:image/png;base64,{_PIXEL_PNG}"], [{"image": _PIXEL_PNG}]), + ], + ids=["text_only", "text_and_image", "image_only"], +) +def test_jina_ai_img_embeddings_transforms_mixed_text_and_image_inputs( + input_data: list[str], expected_payload_input: list[object], respx_mock: respx.MockRouter +) -> None: + route: Final = respx_mock.post("https://api.jina.ai/v1/embeddings").respond( + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "jina-embeddings-v4", + "usage": {"prompt_tokens": 3, "total_tokens": 3}, + } + ) + + response: Final = litellm.embedding(model="jina_ai/jina-embeddings-v4", input=input_data, api_key="jina-test-key") + + assert route.call_count == 1 + assert json.loads(route.calls.last.request.content) == { + "model": "jina-embeddings-v4", + "input": expected_payload_input, + } + assert response.data[0]["embedding"] == [0.1, 0.2] diff --git a/tests/unit/llms/mistral/test_mistral_completion.py b/tests/unit/llms/mistral/test_mistral_completion.py index 7503d0d9665..973e2a46634 100644 --- a/tests/unit/llms/mistral/test_mistral_completion.py +++ b/tests/unit/llms/mistral/test_mistral_completion.py @@ -1,5 +1,10 @@ +import json + import pytest +import respx import litellm +import httpx +from typing import Final @pytest.fixture(autouse=True) @@ -168,3 +173,155 @@ async def test_mistral_transform_request_name_field_removal( assert user_message["role"] == "user" assert user_message["content"] == "Hello" assert "name" not in user_message # The 'name' field should have been removed + + +def test_mistral_401_maps_to_authentication_error(respx_mock): + route = respx_mock.post("https://api.mistral.ai/v1/chat/completions").mock( + return_value=httpx.Response(401, json={"message": "invalid API key"}) + ) + + with pytest.raises(litellm.AuthenticationError) as exc_info: + litellm.completion( + model="mistral/mistral-tiny", + messages=[{"role": "user", "content": "hello"}], + api_key="bad-mistral-key", + max_retries=0, + ) + + assert exc_info.value.status_code == 401 + assert "invalid API key" in str(exc_info.value) + assert route.calls.last.request.headers["Authorization"] == "Bearer bad-mistral-key" + + +def test_mistral_streaming_function_call_arguments_are_assembled( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + chunks: Final = ( + { + "id": "chatcmpl-stream", + "object": "chat.completion.chunk", + "created": 1, + "model": "mistral-medium-latest", + "choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}], + }, + { + "id": "chatcmpl-stream", + "object": "chat.completion.chunk", + "created": 1, + "model": "mistral-medium-latest", + "choices": [ + { + "index": 0, + "delta": { + "tool_calls": [ + { + "index": 0, + "id": "call_weather", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": '{"location":"', + }, + } + ] + }, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-stream", + "object": "chat.completion.chunk", + "created": 1, + "model": "mistral-medium-latest", + "choices": [ + { + "index": 0, + "delta": { + "tool_calls": [ + { + "index": 0, + "function": {"arguments": 'Boston","unit":"fahrenheit"}'}, + } + ] + }, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-stream", + "object": "chat.completion.chunk", + "created": 1, + "model": "mistral-medium-latest", + "choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}], + }, + ) + body: Final = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n" + respx_mock.post("https://api.mistral.ai/v1/chat/completions").respond( + 200, + text=body, + headers={"content-type": "text/event-stream"}, + ) + messages: Final = [{"role": "user", "content": "What is the weather in Boston?"}] + stream: Final = litellm.completion( + model="mistral/mistral-medium-latest", + messages=messages, + tools=[ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + "unit": {"type": "string"}, + }, + "required": ["location", "unit"], + }, + }, + } + ], + stream=True, + ) + stream_chunks: Final = tuple(stream) + assert {chunk._hidden_params["custom_llm_provider"] for chunk in stream_chunks} == {"mistral"} + argument_fragments: Final = tuple( + chunk.choices[0].delta.tool_calls[0].function.arguments + for chunk in stream_chunks + if chunk.choices and chunk.choices[0].delta.tool_calls is not None + ) + assert "".join(argument or "" for argument in argument_fragments) == ( + '{"location":"Boston","unit":"fahrenheit"}' + ) + assembled: Final = litellm.stream_chunk_builder(chunks=list(stream_chunks), messages=messages) + assert assembled is not None + assert assembled.choices[0].message.tool_calls[0].function.name == "get_current_weather" + assert json.loads(assembled.choices[0].message.tool_calls[0].function.arguments) == { + "location": "Boston", + "unit": "fahrenheit", + } + assert assembled.choices[0].finish_reason == "tool_calls" + + +def test_mistral_completion_cost_is_priced_from_the_returned_usage( + respx_mock, mistral_api_response, local_model_cost_map +): + route: Final = respx_mock.post("https://api.mistral.ai/v1/chat/completions").respond(json=mistral_api_response) + + response: Final = litellm.completion( + model="mistral/mistral-medium-latest", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + max_tokens=5, + seed=10, + input_cost_per_token=4e-07, + output_cost_per_token=2e-06, + ) + + body: Final = json.loads(route.calls.last.request.content) + assert (body["random_seed"], body["max_tokens"]) == (10, 5) + assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (10, 15) + assert response._hidden_params["response_cost"] == pytest.approx(10 * 4e-07 + 15 * 2e-06) diff --git a/tests/unit/llms/mistral/test_mistral_embedding.py b/tests/unit/llms/mistral/test_mistral_embedding.py new file mode 100644 index 00000000000..bc284e72176 --- /dev/null +++ b/tests/unit/llms/mistral/test_mistral_embedding.py @@ -0,0 +1,34 @@ +import json +from typing import Final + +import httpx +import respx + +import litellm + + +def test_mistral_embedding_uses_mistral_embeddings_endpoint(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://api.mistral.ai/v1/embeddings").mock( + return_value=httpx.Response( + 200, + json={ + "id": "embd-migration", + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "mistral-embed", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + }, + ) + ) + + response: Final = litellm.embedding( + model="mistral/mistral-embed", + input=["hello"], + api_key="test-mistral-key", + ) + + assert str(route.calls.last.request.url) == "https://api.mistral.ai/v1/embeddings" + assert route.calls.last.request.headers["Authorization"] == "Bearer test-mistral-key" + assert json.loads(route.calls.last.request.read()) == {"model": "mistral-embed", "input": ["hello"]} + assert response.data[0]["embedding"] == [0.1, 0.2] + assert (response.usage.prompt_tokens, response.usage.total_tokens) == (2, 2) diff --git a/tests/unit/llms/ollama/test_ollama_completion_transformation.py b/tests/unit/llms/ollama/test_ollama_completion_transformation.py index 2979cc7d572..7be377247af 100644 --- a/tests/unit/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_completion_transformation.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch import httpx import pytest +import respx import litellm from litellm._uuid import uuid @@ -886,3 +887,36 @@ def test_transform_request_leaves_unreadable_images_untouched(payload: str) -> N data = _transform_image_request(payload, "png") assert data["images"] == [payload] + + +def test_ollama_vision_model_sends_the_image_beside_the_user_prompt(respx_mock: respx.MockRouter) -> None: + image_base64: Final = _image_base64("PNG") + route: Final = respx_mock.post("http://ollama.test:11434/api/generate").respond( + json={ + "model": "llama3.2-vision:11b", + "response": "A white square", + "done": True, + "prompt_eval_count": 5, + "eval_count": 3, + } + ) + + response: Final = litellm.completion( + model="ollama/llama3.2-vision:11b", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Whats in this image?"}, + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_base64}"}}, + ], + } + ], + api_base="http://ollama.test:11434", + ) + + body: Final = json.loads(route.calls.last.request.content) + assert body["model"] == "llama3.2-vision:11b" + assert body["images"] == [image_base64] + assert body["prompt"] == "### User:\nWhats in this image?\n\n" + assert response.choices[0].message.content == "A white square" diff --git a/tests/unit/llms/ollama/test_ollama_text_completion.py b/tests/unit/llms/ollama/test_ollama_text_completion.py new file mode 100644 index 00000000000..a170fc9b64f --- /dev/null +++ b/tests/unit/llms/ollama/test_ollama_text_completion.py @@ -0,0 +1,39 @@ +import json +from typing import Final + +import httpx +import respx + +from litellm import text_completion + + +@respx.mock +def test_ollama_text_completion_sends_prompt() -> None: + endpoint: Final = "http://localhost:11434/api/generate" + route: Final = respx.post(endpoint).mock( + return_value=httpx.Response( + 200, + json={ + "model": "llama3.1", + "created_at": "2024-01-01T00:00:00Z", + "response": "Hello!", + "done": True, + "context": [1], + "total_duration": 1, + "load_duration": 1, + "prompt_eval_count": 2, + "eval_count": 1, + }, + ) + ) + + response: Final = text_completion( + model="ollama/llama3.1", + prompt="hello", + api_base="http://localhost:11434", + ) + + request_body: Final = json.loads(route.calls.last.request.read()) + assert request_body["model"] == "llama3.1" + assert request_body["prompt"] == "hello" + assert response.choices[0].text == "Hello!" diff --git a/tests/unit/llms/openai/completion/test_completion_handler.py b/tests/unit/llms/openai/completion/test_completion_handler.py index 5c0f633430e..f465bbea663 100644 --- a/tests/unit/llms/openai/completion/test_completion_handler.py +++ b/tests/unit/llms/openai/completion/test_completion_handler.py @@ -6,6 +6,9 @@ Regression tests for https://github.com/BerriAI/litellm/issues/27410 """ +import json +from typing import Final + import pytest import respx from httpx import Response @@ -147,3 +150,227 @@ async def test_async_streaming_completion_reports_an_error_event_sent_mid_stream assert failed.value.status_code == 500 assert "upstream overloaded" in str(failed.value) + + +@respx.mock +def test_completion_openai_prompt_array_sends_both_prompts() -> None: + route: Final = respx.post("https://api.openai.com/v1/completions").mock( + return_value=Response( + 200, + json={ + "id": "cmpl-test123", + "object": "text_completion", + "created": 1677652288, + "model": "gpt-3.5-turbo-instruct", + "choices": [ + {"text": "first answer", "index": 0, "logprobs": None, "finish_reason": "stop"}, + {"text": "second answer", "index": 1, "logprobs": None, "finish_reason": "stop"}, + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 2, "total_tokens": 4}, + }, + ) + ) + + response: Final = text_completion( + model="gpt-3.5-turbo-instruct", + prompt=["first prompt", "second prompt"], + max_tokens=5, + ) + + body: Final = json.loads(route.calls.last.request.read()) + assert body["prompt"] == ["first prompt", "second prompt"] + assert [choice.text for choice in response.choices] == ["first answer", "second answer"] + + +@respx.mock +def test_completion_text_003_token_prompt_array_returns_one_choice_per_prompt() -> None: + token_prompts: Final = [[2061, 338, 262, 6193, 287, 14362, 30], [2437, 318, 9502, 30]] + route: Final = respx.post("https://api.fireworks.ai/inference/v1/completions").mock( + return_value=Response( + 200, + json={ + "id": "cmpl-token-prompts", + "object": "text_completion", + "created": 1677652288, + "model": "accounts/fireworks/models/glm-5p3-flash", + "choices": [ + {"text": "sunny", "index": 0, "logprobs": None, "finish_reason": "length"}, + {"text": "rainy", "index": 1, "logprobs": None, "finish_reason": "length"}, + ], + "usage": {"prompt_tokens": 11, "completion_tokens": 2, "total_tokens": 13}, + }, + ) + ) + + response: Final = text_completion( + model="text-completion-openai/accounts/fireworks/models/glm-5p3-flash", + api_base="https://api.fireworks.ai/inference/v1", + api_key="fireworks-test-key", + prompt=token_prompts, + max_tokens=5, + ) + + body: Final = json.loads(route.calls.last.request.read()) + assert body["prompt"] == token_prompts + assert body["model"] == "accounts/fireworks/models/glm-5p3-flash" + assert [(choice.index, choice.text) for choice in response.choices] == [(0, "sunny"), (1, "rainy")] + + +@respx.mock +def test_text_completion_with_echo_returns_prompt_and_token_logprobs() -> None: + route: Final = respx.post("https://api.openai.com/v1/completions").mock( + return_value=Response( + 200, + json={ + "id": "cmpl-test123", + "object": "text_completion", + "created": 1677652288, + "model": "gpt-3.5-turbo-instruct", + "choices": [ + { + "text": "hello world", + "index": 0, + "logprobs": {"tokens": ["hello", " world"], "token_logprobs": [-0.1, -0.2]}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + ) + + response: Final = text_completion( + model="gpt-3.5-turbo-instruct", + prompt="hello", + max_tokens=1, + echo=True, + logprobs=1, + ) + + body: Final = json.loads(route.calls.last.request.read()) + assert body["echo"] is True + assert response.choices[0].text == "hello world" + assert response.choices[0].logprobs.token_logprobs == [-0.1, -0.2] + + +@respx.mock +def test_completion_openai_engine_selects_the_engine_model(mock_completions_endpoint) -> None: + response: Final = text_completion(engine="gpt-3.5-turbo-instruct", prompt="hello", max_tokens=5) + + request_body: Final = json.loads(mock_completions_endpoint.calls.last.request.read()) + assert request_body["model"] == "gpt-3.5-turbo-instruct" + assert response.model == "gpt-3.5-turbo-instruct" + + +@respx.mock +def test_completion_openai_engine_and_model_uses_the_explicit_model(mock_completions_endpoint) -> None: + response: Final = text_completion( + model="gpt-3.5-turbo-instruct", + engine="unused-engine", + prompt="hello", + max_tokens=5, + ) + + request_body: Final = json.loads(mock_completions_endpoint.calls.last.request.read()) + assert request_body["model"] == "gpt-3.5-turbo-instruct" + assert response.model == "gpt-3.5-turbo-instruct" + + +@respx.mock +async def test_async_text_completion_stream_returns_text_and_one_finish_reason(monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx.post("https://api.openai.com/v1/completions").mock( + return_value=Response( + 200, + text=( + 'data: {"id":"cmpl-stream","object":"text_completion","created":1,"model":"gpt-3.5-turbo-instruct","choices":[{"index":0,"text":"hello","finish_reason":null}]}\n\n' + 'data: {"id":"cmpl-stream","object":"text_completion","created":1,"model":"gpt-3.5-turbo-instruct","choices":[{"index":0,"text":"","finish_reason":"stop"}]}\n\n' + "data: [DONE]\n\n" + ), + headers={"content-type": "text/event-stream"}, + ) + ) + + stream: Final = await atext_completion( + model="gpt-3.5-turbo-instruct", prompt="say hello", stream=True, max_tokens=5 + ) + chunks: Final = [chunk async for chunk in stream] + + assert [chunk.choices[0].text for chunk in chunks if chunk.choices[0].text] == ["hello"] + assert tuple( + chunk.choices[0].finish_reason + for chunk in chunks + if chunk.choices[0].finish_reason is not None + ) == ("stop",) + assert len(route.calls) == 1 + + +@respx.mock +async def test_async_text_completion_chat_model_stream_builds_chat_chunks(monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=Response( + 200, + text=( + 'data: {"id":"chatcmpl-stream","object":"chat.completion.chunk","created":1,"model":"gpt-3.5-turbo","choices":[{"index":0,"delta":{"role":"assistant","content":"hello"},"finish_reason":null}]}\n\n' + 'data: {"id":"chatcmpl-stream","object":"chat.completion.chunk","created":1,"model":"gpt-3.5-turbo","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n' + "data: [DONE]\n\n" + ), + headers={"content-type": "text/event-stream"}, + ) + ) + + stream: Final = await atext_completion( + model="gpt-3.5-turbo", prompt="say hello", stream=True, max_tokens=5 + ) + chunks: Final = [chunk async for chunk in stream] + response: Final = litellm.stream_chunk_builder(chunks=chunks) + + assert response.choices[0].text == "hello" + assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (0, 1) + assert litellm.completion_cost( + completion_response=response, + custom_cost_per_token={"input_cost_per_token": 5e-07, "output_cost_per_token": 1.5e-06}, + ) == pytest.approx(1 * 1.5e-06) + assert tuple( + chunk.choices[0].finish_reason + for chunk in chunks + if chunk.choices[0].finish_reason is not None + ) == ("stop",) + assert len(route.calls) == 1 + + +@respx.mock +def test_text_completion_stream_forwards_stream_options_and_usage(monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx.post("https://api.openai.com/v1/completions").mock( + return_value=Response( + 200, + text=( + 'data: {"id":"cmpl-stream","object":"text_completion","created":1,"model":"gpt-3.5-turbo-instruct","choices":[{"index":0,"text":"hello","finish_reason":null}]}\n\n' + 'data: {"id":"cmpl-stream","object":"text_completion","created":1,"model":"gpt-3.5-turbo-instruct","choices":[],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}\n\n' + "data: [DONE]\n\n" + ), + headers={"content-type": "text/event-stream"}, + ) + ) + + chunks: Final = list( + text_completion( + model="gpt-3.5-turbo-instruct", + prompt="say hello", + stream=True, + stream_options={"include_usage": True}, + ) + ) + + assert tuple(chunk.choices[0].text for chunk in chunks) == ("hello", None, None) + assert tuple(chunk.usage for chunk in chunks[:-1]) == (None, None) + assert (chunks[-1].usage.prompt_tokens, chunks[-1].usage.completion_tokens, chunks[-1].usage.total_tokens) == ( + 2, + 1, + 3, + ) + assert json.loads(route.calls[0].request.content)["stream_options"] == { + "include_usage": True + } diff --git a/tests/unit/llms/openai/test_openai_authentication_error.py b/tests/unit/llms/openai/test_openai_authentication_error.py new file mode 100644 index 00000000000..65a0d7368c0 --- /dev/null +++ b/tests/unit/llms/openai/test_openai_authentication_error.py @@ -0,0 +1,79 @@ +from typing import Final + +import httpx +import pytest +import respx + +import litellm + + +def test_openai_401_maps_to_authentication_error(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 401, + json={ + "error": { + "message": "Incorrect API key provided", + "type": "invalid_request_error", + "param": None, + "code": "invalid_api_key", + } + }, + ) + ) + + with pytest.raises(litellm.AuthenticationError) as error: + litellm.completion( + model="openai/migration-auth-error-model", + messages=[{"role": "user", "content": "hello"}], + api_key="invalid-test-key", + max_retries=0, + ) + + assert error.value.status_code == 401 + assert str(error.value).startswith("litellm.AuthenticationError: AuthenticationError: OpenAIException - ") + assert "Incorrect API key provided" in str(error.value) + assert route.calls.last.request.headers["Authorization"] == "Bearer invalid-test-key" + + +async def _complete(sync_mode: bool, request: dict[str, object]) -> object: + if sync_mode: + return litellm.completion(**request) + return await litellm.acompletion(**request) + + +@pytest.mark.parametrize("stream", [True, False]) +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_openai_invalid_role_error_preserves_provider_fields( + sync_mode: bool, stream: bool, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 400, + json={ + "error": { + "message": "Invalid value: 'usera'. Supported values are: 'system', 'assistant', 'user'.", + "type": "invalid_request_error", + "param": "messages[0].role", + "code": "invalid_value", + } + }, + ) + ) + request: Final = { + "model": "openai/migration-invalid-role-model", + "messages": [{"role": "usera", "content": "hi"}], + "api_key": "test-key", + "max_retries": 0, + "stream": stream, + } + + with pytest.raises(litellm.BadRequestError) as error: + await _complete(sync_mode, request) + + assert str(error.value).startswith("litellm.BadRequestError: OpenAIException - Invalid value") + assert error.value.code == "invalid_value" + assert error.value.param == "messages[0].role" + assert error.value.type == "invalid_request_error" + assert error.value.response.status_code == 400 diff --git a/tests/unit/llms/openai_like/embedding/test_openai_like_embedding.py b/tests/unit/llms/openai_like/embedding/test_openai_like_embedding.py index 89348d505bb..e413677dadc 100644 --- a/tests/unit/llms/openai_like/embedding/test_openai_like_embedding.py +++ b/tests/unit/llms/openai_like/embedding/test_openai_like_embedding.py @@ -3,9 +3,13 @@ Test cases for OpenAI-like embedding handler """ import json +from typing import Final from unittest.mock import MagicMock, Mock, patch import pytest +import respx + +import litellm from litellm.llms.openai_like.embedding.handler import OpenAILikeEmbeddingHandler from litellm.types.utils import EmbeddingResponse @@ -346,3 +350,40 @@ class TestOpenAILikeEmbeddingHandler: assert sent_data["model"] == "test-model" assert sent_data["input"] == ["test input"] assert "encoding_format" not in sent_data + + +_OPENAI_LIKE_EMBEDDING_BODY: Final = { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.5, 0.25]}], + "model": "jina-embeddings-v3", + "usage": {"prompt_tokens": 2, "total_tokens": 2}, +} + + +def test_llamafile_embedding_posts_to_the_env_api_base( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("LLAMAFILE_API_BASE", "http://llamafile.test/v1") + route: Final = respx_mock.post("http://llamafile.test/v1/embeddings").respond(json=_OPENAI_LIKE_EMBEDDING_BODY) + + response: Final = litellm.embedding(model="llamafile/jina-embeddings-v3", input=["Hello world"]) + + assert route.call_count == 1 + assert json.loads(route.calls.last.request.content) == {"model": "jina-embeddings-v3", "input": ["Hello world"]} + assert response.data[0]["embedding"] == [0.5, 0.25] + + +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_lm_studio_embedding_posts_to_the_env_api_base( + sync_mode: bool, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("LM_STUDIO_API_BASE", "http://lm-studio.test") + route: Final = respx_mock.post("http://lm-studio.test/embeddings").respond(json=_OPENAI_LIKE_EMBEDDING_BODY) + request: Final = {"model": "lm_studio/jina-embeddings-v3", "input": ["Hello world"]} + + response: Final = litellm.embedding(**request) if sync_mode else await litellm.aembedding(**request) + + assert route.call_count == 1 + assert json.loads(route.calls.last.request.content) == {"model": "jina-embeddings-v3", "input": ["Hello world"]} + assert response.data[0]["embedding"] == [0.5, 0.25] diff --git a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py index beffcf9b69c..32b47c1c680 100644 --- a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -1,6 +1,12 @@ +import json +from typing import Final + import httpx import pytest +import respx + +import litellm from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig @@ -620,3 +626,139 @@ def test_openrouter_reasoning_effort_high_passes_through(): ) assert result["reasoning_effort"] == "high" + + +@respx.mock +def test_openrouter_streaming_completion_assembles_text_and_finish_reason() -> None: + chunks: Final = ( + { + "id": "gen-stream-model", + "object": "chat.completion.chunk", + "created": 1, + "model": "openai/gpt-4o-mini", + "choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}], + }, + { + "id": "gen-stream-model", + "object": "chat.completion.chunk", + "created": 1, + "model": "openai/gpt-4o-mini", + "choices": [{"index": 0, "delta": {"content": "The Supreme Court"}, "finish_reason": None}], + }, + { + "id": "gen-stream-model", + "object": "chat.completion.chunk", + "created": 1, + "model": "openai/gpt-4o-mini", + "choices": [{"index": 0, "delta": {"content": " hears federal cases."}, "finish_reason": None}], + }, + { + "id": "gen-stream-model", + "object": "chat.completion.chunk", + "created": 1, + "model": "openai/gpt-4o-mini", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + ) + body: Final = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n" + respx.post("https://openrouter.ai/api/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + text=body, + headers={"content-type": "text/event-stream"}, + ) + ) + messages: Final = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "How does a case reach the Supreme Court?"}, + ] + stream: Final = litellm.completion( + model="openrouter/openai/gpt-4o-mini", + messages=messages, + stream=True, + max_tokens=50, + api_key="test-openrouter-key", + ) + stream_chunks: Final = tuple(stream) + assert stream_chunks[0].choices[0].delta.role == "assistant" + assert tuple( + chunk.choices[0].finish_reason for chunk in stream_chunks if chunk.choices[0].finish_reason is not None + ) == ("stop",) + assert "".join( + chunk.choices[0].delta.content or "" for chunk in stream_chunks if chunk.choices + ) == "The Supreme Court hears federal cases." + assembled: Final = litellm.stream_chunk_builder(chunks=list(stream_chunks), messages=messages) + assert assembled is not None + assert assembled.choices[0].message.content == "The Supreme Court hears federal cases." + assert assembled.choices[0].finish_reason == "stop" + + +@respx.mock +def test_openrouter_streaming_reasoning_is_preserved() -> None: + chunks: Final = ( + { + "id": "gen-reasoning-stream", + "object": "chat.completion.chunk", + "created": 1, + "model": "anthropic/claude-sonnet-4.5", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "reasoning": "Consider the evidence. "}, + "finish_reason": None, + } + ], + }, + { + "id": "gen-reasoning-stream", + "object": "chat.completion.chunk", + "created": 1, + "model": "anthropic/claude-sonnet-4.5", + "choices": [ + { + "index": 0, + "delta": {"content": "The decision is reasoned.", "reasoning": "Apply the rule."}, + "finish_reason": None, + } + ], + }, + { + "id": "gen-reasoning-stream", + "object": "chat.completion.chunk", + "created": 1, + "model": "anthropic/claude-sonnet-4.5", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + ) + body: Final = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n" + route: Final = respx.post("https://openrouter.ai/api/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + text=body, + headers={"content-type": "text/event-stream"}, + ) + ) + parsed_reasoning_chunk: Final = OpenRouterChatCompletionStreamingHandler( + streaming_response=None, sync_stream=True + ).chunk_parser(chunks[0]) + assert parsed_reasoning_chunk.choices[0]["delta"]["reasoning_content"] == "Consider the evidence. " + messages: Final = [{"role": "user", "content": "Explain the decision briefly."}] + stream: Final = litellm.completion( + model="openrouter/anthropic/claude-sonnet-4.5", + messages=messages, + stream=True, + reasoning={"effort": "high"}, + drop_params=True, + api_key="test-openrouter-key", + ) + stream_chunks: Final = tuple(stream) + assert json.loads(route.calls.last.request.content)["reasoning"] == {"effort": "high"} + reasoning_deltas: Final = tuple( + getattr(chunk.choices[0].delta, "reasoning_content", None) + for chunk in stream_chunks + if chunk.choices + ) + assert reasoning_deltas == ("Consider the evidence. ", "Apply the rule.", None) + assembled: Final = litellm.stream_chunk_builder(chunks=list(stream_chunks), messages=messages) + assert assembled is not None + assert assembled.choices[0].message.reasoning_content == "Consider the evidence. Apply the rule." diff --git a/tests/unit/llms/perplexity/test_perplexity.py b/tests/unit/llms/perplexity/test_perplexity.py index 797a56070c6..9401fd7dc8a 100644 --- a/tests/unit/llms/perplexity/test_perplexity.py +++ b/tests/unit/llms/perplexity/test_perplexity.py @@ -1,6 +1,10 @@ import pytest +import httpx +import respx + +import litellm class TestPerplexityWebSearch: @@ -19,3 +23,22 @@ class TestPerplexityWebSearch: assert ( "web_search_options" in supported_params ), f"web_search_options should be supported for {model}" + + +def test_perplexity_401_maps_to_authentication_error(respx_mock: respx.MockRouter) -> None: + route = respx_mock.post("https://api.perplexity.ai/chat/completions").mock( + return_value=httpx.Response(401, json={"error": {"message": "invalid API key"}}) + ) + + with pytest.raises(litellm.AuthenticationError) as exc_info: + litellm.completion( + model="perplexity/mistral-7b-instruct", + messages=[{"role": "user", "content": "hello"}], + api_key="bad-perplexity-key", + max_retries=0, + ) + + assert exc_info.value.status_code == 401 + assert "PerplexityException" in str(exc_info.value) + assert "invalid API key" in str(exc_info.value) + assert route.calls.last.request.headers["Authorization"] == "Bearer bad-perplexity-key" diff --git a/tests/unit/llms/replicate/chat/test_transformation.py b/tests/unit/llms/replicate/chat/test_transformation.py index 0270d86d485..549e915a6d5 100644 --- a/tests/unit/llms/replicate/chat/test_transformation.py +++ b/tests/unit/llms/replicate/chat/test_transformation.py @@ -1,4 +1,5 @@ import json +from typing import Final from unittest.mock import AsyncMock, Mock, patch import httpx @@ -272,3 +273,29 @@ def test_sync_completion_handles_starting_status(mock_get_client, monkeypatch, r assert json.loads(mock_client.post.call_args.kwargs["data"])["input"]["prompt"] == ( "<｜begin▁of▁sentence｜><｜User｜>Hi<｜Assistant｜>" ) + + +@pytest.mark.parametrize( + ("model", "max_tokens_key"), + [ + ("replicate/vicuna-13b", "max_length"), + ("replicate/flan-t5-xl", "max_length"), + ("meta/codellama-13b", "max_tokens"), + ("meta/llama-2-7b", "max_new_tokens"), + ], +) +def test_map_openai_params_renames_openai_params_to_replicate_inputs(model: str, max_tokens_key: str): + mapped: Final = ReplicateConfig().map_openai_params( + non_default_params={"stream": True, "max_tokens": 64, "temperature": 0.2, "top_p": 0.9, "stop": ["\n"]}, + optional_params={}, + model=model, + drop_params=False, + ) + + assert mapped == { + "stream": True, + max_tokens_key: 64, + "temperature": 0.2, + "top_p": 0.9, + "stop_sequences": ["\n"], + } diff --git a/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py b/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py index 51bdf5d6848..f8caa8d5fed 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py @@ -14,7 +14,7 @@ import binascii import json import struct from collections.abc import Mapping -from typing import AsyncIterator, Iterator +from typing import AsyncIterator, Final, Iterator from unittest.mock import MagicMock import httpx @@ -400,3 +400,33 @@ def test_get_complete_url_sagemaker_base_url_override_wins() -> None: stream=False, ) assert url == "https://my-private-endpoint.example.com/invocations" + + +def test_sagemaker_default_region_is_us_west_2(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("AWS_REGION_NAME", raising=False) + monkeypatch.delenv("AWS_REGION", raising=False) + monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False) + monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "true") + url: Final = SagemakerChatConfig().get_complete_url( + api_base=None, + api_key=None, + model="my-endpoint", + optional_params={}, + litellm_params={}, + stream=False, + ) + + assert url == "https://runtime.sagemaker.us-west-2.amazonaws.com/endpoints/my-endpoint/invocations" + + +def test_sagemaker_explicit_region_overrides_the_default() -> None: + url: Final = SagemakerChatConfig().get_complete_url( + api_base=None, + api_key=None, + model="my-endpoint", + optional_params={"aws_region_name": "us-east-1"}, + litellm_params={}, + stream=False, + ) + + assert url == "https://runtime.sagemaker.us-east-1.amazonaws.com/endpoints/my-endpoint/invocations" diff --git a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py index 339e02c20d5..0980c9f5c66 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py @@ -12,16 +12,26 @@ inflating TTFT and turning a steady provider stream into gap-then-burst delivery import binascii import json import struct -from typing import AsyncIterator, Iterator +from typing import AsyncIterator, Final, Iterator from unittest.mock import MagicMock import httpx +import litellm import pytest +import respx from litellm.llms.sagemaker.common_utils import SagemakerError from litellm.llms.sagemaker.completion.handler import SagemakerLLM +@pytest.fixture +def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + def _encode_header(name: str, value: str) -> bytes: name_b = name.encode("utf-8") value_b = value.encode("utf-8") @@ -293,3 +303,318 @@ def test_installed_botocore_signs_the_completion_request(): ) assert request.body == b'{"inputs": "ping"}' assert "/us-west-2/sagemaker/aws4_request" in request.headers["Authorization"] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.usefixtures("fake_provider_credentials") +async def test_acompletion_sagemaker_non_stream( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("AWS_REGION_NAME", raising=False) + monkeypatch.delenv("AWS_REGION", raising=False) + url: Final = ( + "https://runtime.sagemaker.us-west-2.amazonaws.com/endpoints/" + "jumpstart-dft-hf-textgeneration1-mp-20240815-185614/invocations" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response(200, json={"generated_text": "SageMaker reply"}) + ) + response: Final = await litellm.acompletion( + model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + messages=[{"role": "user", "content": "hi"}], + temperature=0.2, + max_tokens=80, + cost_per_second=0.000420, + num_retries=0, + ) + + assert route.called + assert str(route.calls[0].request.url) == url + assert json.loads(route.calls[0].request.content) == { + "inputs": "hi", + "parameters": {"temperature": 0.2, "max_new_tokens": 80}, + } + assert response.choices[0].message.content == "SageMaker reply" + + +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.usefixtures("fake_provider_credentials") +def test_completion_sagemaker_non_stream( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("AWS_REGION_NAME", raising=False) + monkeypatch.delenv("AWS_REGION", raising=False) + url: Final = ( + "https://runtime.sagemaker.us-west-2.amazonaws.com/endpoints/" + "jumpstart-dft-hf-textgeneration1-mp-20240815-185614/invocations" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response(200, json={"generated_text": "SageMaker reply"}) + ) + response: Final = litellm.completion( + model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + messages=[{"role": "user", "content": "hi"}], + temperature=0.2, + max_tokens=80, + cost_per_second=0.000420, + num_retries=0, + ) + + assert route.called + assert str(route.calls[0].request.url) == url + assert json.loads(route.calls[0].request.content) == { + "inputs": "hi", + "parameters": {"temperature": 0.2, "max_new_tokens": 80}, + } + assert response.choices[0].message.content == "SageMaker reply" + + +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.usefixtures("fake_provider_credentials") +def test_completion_sagemaker_non_stream_with_aws_params(respx_mock: respx.MockRouter) -> None: + url: Final = ( + "https://runtime.sagemaker.us-west-5.amazonaws.com/endpoints/" + "jumpstart-dft-hf-textgeneration1-mp-20240815-185614/invocations" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response(200, json={"generated_text": "SageMaker reply"}) + ) + response: Final = litellm.completion( + model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + messages=[{"role": "user", "content": "hi"}], + temperature=0.2, + max_tokens=80, + cost_per_second=0.000420, + aws_access_key_id="unit-test-access-key", + aws_secret_access_key="unit-test-secret", + aws_region_name="us-west-5", + num_retries=0, + ) + + assert route.called + assert str(route.calls[0].request.url) == url + assert json.loads(route.calls[0].request.content) == { + "inputs": "hi", + "parameters": {"temperature": 0.2, "max_new_tokens": 80}, + } + assert response.choices[0].message.content == "SageMaker reply" + + +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.usefixtures("fake_provider_credentials") +def test_completion_sagemaker_prompt_template_non_stream( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "known_tokenizer_config", {}) + tokenizer_route: Final = respx_mock.get( + "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct/raw/main/tokenizer_config.json" + ).mock( + return_value=httpx.Response( + 200, + json={ + "bos_token": {"content": "<｜begin▁of▁sentence｜>"}, + "eos_token": {"content": "<|EOT|>"}, + "chat_template": ( + "{{ bos_token }}You are a coding assistant\n" + "{% for message in messages %}### Instruction:\n{{ message['content'] }}\n{% endfor %}" + "### Response:" + ), + }, + ) + ) + url: Final = ( + "https://runtime.sagemaker.us-east-1.amazonaws.com/endpoints/deepseek_coder_6.7_instruct/invocations" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response(200, json={"generated_text": "SageMaker reply"}) + ) + response: Final = litellm.completion( + model="sagemaker/deepseek_coder_6.7_instruct", + messages=[{"role": "user", "content": "hi"}], + temperature=0.2, + max_tokens=80, + hf_model_name="deepseek-ai/deepseek-coder-6.7b-instruct", + aws_region_name="us-east-1", + num_retries=0, + ) + + assert tokenizer_route.call_count == 1 + assert json.loads(route.calls[0].request.content) == { + "inputs": "<｜begin▁of▁sentence｜>You are a coding assistant\n### Instruction:\nhi\n### Response:", + "parameters": {"temperature": 0.2, "max_new_tokens": 80}, + } + assert response.choices[0].message.content == "SageMaker reply" + + +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.usefixtures("fake_provider_credentials") +@pytest.mark.parametrize( + ("environment", "call_kwargs", "expected_region"), + [ + ({}, {}, "us-west-2"), + ({"AWS_REGION_NAME": "us-east-1"}, {}, "us-east-1"), + ({"AWS_REGION_NAME": "eu-west-1"}, {"aws_region_name": "us-east-1"}, "us-east-1"), + ], + ids=["default_region", "environment_region", "config_region_beats_environment"], +) +def test_sagemaker_region_resolution( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + environment: dict[str, str], + call_kwargs: dict[str, str], + expected_region: str, +) -> None: + monkeypatch.delenv("AWS_REGION_NAME", raising=False) + monkeypatch.delenv("AWS_REGION", raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + url: Final = f"https://runtime.sagemaker.{expected_region}.amazonaws.com/endpoints/mock-endpoint/invocations" + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response(200, json={"generated_text": "SageMaker reply"}) + ) + litellm.completion( + model="sagemaker/mock-endpoint", + messages=[{"role": "user", "content": "Hello, world!"}], + num_retries=0, + **call_kwargs, + ) + + assert route.call_count == 1 + assert str(route.calls[0].request.url) == url + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.usefixtures("fake_provider_credentials") +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_completion_sagemaker( + respx_mock: respx.MockRouter, + sync_mode: bool, +) -> None: + url: Final = ( + "https://runtime.sagemaker.us-east-1.amazonaws.com/endpoints/" + "jumpstart-dft-hf-textgeneration1-mp-20240815-185614/invocations" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response(200, json={"generated_text": "SageMaker reply"}) + ) + rate: Final = 0.000420 + response: Final = ( + litellm.completion( + model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + messages=[{"role": "user", "content": "hi"}], + temperature=0.2, + max_tokens=80, + cost_per_second=rate, + aws_region_name="us-east-1", + num_retries=0, + ) + if sync_mode + else await litellm.acompletion( + model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + messages=[{"role": "user", "content": "hi"}], + temperature=0.2, + max_tokens=80, + cost_per_second=rate, + aws_region_name="us-east-1", + num_retries=0, + ) + ) + response._response_ms = 1000.0 + + assert route.called + assert response.choices[0].message.content == "SageMaker reply" + assert litellm.completion_cost(completion_response=response) == pytest.approx(rate) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.usefixtures("fake_provider_credentials") +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_completion_sagemaker_stream( + respx_mock: respx.MockRouter, + sync_mode: bool, +) -> None: + url: Final = ( + "https://runtime.sagemaker.us-east-1.amazonaws.com/endpoints/" + "jumpstart-dft-hf-textgeneration1-mp-20240815-185614/invocations-response-stream" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response(200, content=_token_frame("hello") + _token_frame(" world")) + ) + stream: Final = ( + litellm.completion( + model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + messages=[{"role": "user", "content": "hi - what is ur name"}], + stream=True, + temperature=0.2, + max_tokens=80, + cost_per_second=0.000420, + aws_region_name="us-east-1", + num_retries=0, + ) + if sync_mode + else await litellm.acompletion( + model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + messages=[{"role": "user", "content": "hi - what is ur name"}], + stream=True, + temperature=0.2, + max_tokens=80, + cost_per_second=0.000420, + aws_region_name="us-east-1", + num_retries=0, + ) + ) + chunks: Final = tuple(stream) if sync_mode else tuple([chunk async for chunk in stream]) + + assert json.loads(route.calls[0].request.content) == { + "inputs": "hi - what is ur name", + "parameters": {"temperature": 0.2, "max_new_tokens": 80}, + "stream": True, + } + assert chunks[0].choices[0].delta.role == "assistant" + assert tuple(chunk.choices[0].delta.content for chunk in chunks) == ("hello", " world", None) + assert tuple(chunk.choices[0].finish_reason for chunk in chunks) == (None, None, "stop") + + +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.usefixtures("fake_provider_credentials") +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_completion_sagemaker_streaming_bad_request( + respx_mock: respx.MockRouter, + sync_mode: bool, +) -> None: + url: Final = ( + "https://runtime.sagemaker.us-east-1.amazonaws.com/endpoints/" + "jumpstart-dft-hf-textgeneration1-mp-20240815-185614/invocations-response-stream" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response( + 400, + json={"Message": "The request payload failed validation.", "__type": "ValidationException"}, + ) + ) + + with pytest.raises(litellm.BadRequestError) as exc_info: + litellm.completion( + model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + messages=[{"role": "user", "content": "hi"}], + stream=True, + aws_region_name="us-east-1", + num_retries=0, + ) if sync_mode else await litellm.acompletion( + model="sagemaker/jumpstart-dft-hf-textgeneration1-mp-20240815-185614", + messages=[{"role": "user", "content": "hi"}], + stream=True, + aws_region_name="us-east-1", + num_retries=0, + ) + + assert route.called + assert exc_info.value.status_code == 400 + assert exc_info.value.llm_provider == "sagemaker" diff --git a/tests/unit/llms/sagemaker/test_sagemaker_nova_transformation.py b/tests/unit/llms/sagemaker/test_sagemaker_nova_transformation.py index d878bc70a09..12690d5822b 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_nova_transformation.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_nova_transformation.py @@ -2,14 +2,57 @@ Unit tests for SageMaker Nova transformation config. """ +import binascii import json -import pytest +import struct +from typing import Final +import httpx +import litellm +import pytest +import respx + +from litellm.llms.sagemaker.common_utils import AWSEventStreamDecoder from litellm.llms.sagemaker.nova.transformation import SagemakerNovaConfig +from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse from litellm.utils import convert_to_model_response_object +def _event_frame(payload: bytes) -> bytes: + headers: Final = { + ":event-type": "PayloadPart", + ":content-type": "application/json", + ":message-type": "event", + } + encoded_headers: Final = b"".join( + struct.pack("B", len(name.encode())) + + name.encode() + + struct.pack("B", 7) + + struct.pack(">H", len(value.encode())) + + value.encode() + for name, value in headers.items() + ) + prelude: Final = struct.pack(">I", 16 + len(encoded_headers) + len(payload)) + struct.pack( + ">I", len(encoded_headers) + ) + prefix: Final = prelude + struct.pack(">I", binascii.crc32(prelude) & 0xFFFFFFFF) + message: Final = prefix + encoded_headers + payload + return message + struct.pack(">I", binascii.crc32(message) & 0xFFFFFFFF) + + +def _nova_stream_frame(content: str, finish_reason: str | None) -> bytes: + chunk: Final = { + "id": "chatcmpl-nova-stream", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "nova-endpoint", + "choices": [{"index": 0, "delta": {"content": content}, "finish_reason": finish_reason}], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + } + return _event_frame(f"data: {json.dumps(chunk)}\n\n".encode()) + + class TestSagemakerNovaConfig: def setup_method(self): self.config = SagemakerNovaConfig() @@ -394,6 +437,23 @@ class TestSagemakerNovaTransformRequest: assert "messages" in request assert request["temperature"] == 0.7 + def test_should_preserve_system_and_multi_turn_messages_in_request(self): + messages: Final[list[AllMessageValues]] = [ + {"role": "system", "content": "Remember the user's name"}, + {"role": "user", "content": "My name is Alice"}, + {"role": "assistant", "content": "I will remember"}, + {"role": "user", "content": "What is my name?"}, + ] + request = self.config.transform_request( + model="my-nova-endpoint", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert request == {"messages": messages} + def test_should_include_all_nova_params_in_request(self): """Nova-specific params should appear in the request body.""" request = self.config.transform_request( @@ -411,3 +471,82 @@ class TestSagemakerNovaTransformRequest: assert request["top_k"] == 40 assert request["max_tokens"] == 512 assert request["reasoning_effort"] == "low" + + +NOVA_URL: Final = "https://runtime.sagemaker.us-east-1.amazonaws.com/endpoints/nova-endpoint/invocations" + + +@pytest.mark.usefixtures("fake_provider_credentials") +def test_nova_completion_parses_content_finish_reason_and_usage(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(NOVA_URL).mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-nova-single", + "object": "chat.completion", + "created": 1700000000, + "model": "nova-endpoint", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Four"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 14, "completion_tokens": 2, "total_tokens": 16}, + }, + ) + ) + response: Final = litellm.completion( + model="sagemaker_nova/nova-endpoint", + messages=[{"role": "user", "content": "What is 2+2? Reply in one word."}], + max_tokens=32, + temperature=0.1, + num_retries=0, + ) + + assert json.loads(route.calls[0].request.content) == { + "messages": [{"role": "user", "content": "What is 2+2? Reply in one word."}], + "max_tokens": 32, + "temperature": 0.1, + } + assert response.choices[0].message.content == "Four" + assert response.choices[0].finish_reason == "stop" + assert (response.usage.prompt_tokens, response.usage.completion_tokens, response.usage.total_tokens) == ( + 14, + 2, + 16, + ) + + +@pytest.mark.usefixtures("fake_provider_credentials") +def test_nova_streaming_sends_stream_flag_and_parses_chunks_and_usage(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(f"{NOVA_URL}-response-stream").mock( + return_value=httpx.Response( + 200, + content=_nova_stream_frame("Nova ", None) + _nova_stream_frame("stream", "stop"), + ) + ) + stream: Final = litellm.completion( + model="sagemaker_nova/nova-endpoint", + messages=[{"role": "user", "content": "Count from 1 to 5."}], + max_tokens=64, + stream=True, + stream_options={"include_usage": True}, + num_retries=0, + ) + chunks: Final = tuple(stream) + + assert json.loads(route.calls[0].request.content) == { + "messages": [{"role": "user", "content": "Count from 1 to 5."}], + "max_tokens": 64, + "stream": True, + "stream_options": {"include_usage": True}, + } + assert tuple(chunk.choices[0].delta.content for chunk in chunks) == ("Nova ", "stream", None, None) + assert tuple(chunk.choices[0].finish_reason for chunk in chunks) == (None, None, "stop", None) + assert (chunks[-1].usage.prompt_tokens, chunks[-1].usage.completion_tokens, chunks[-1].usage.total_tokens) == ( + 2, + 1, + 3, + ) diff --git a/tests/unit/llms/test_predibase_transformation.py b/tests/unit/llms/test_predibase_transformation.py index 1600878a586..d6425488be3 100644 --- a/tests/unit/llms/test_predibase_transformation.py +++ b/tests/unit/llms/test_predibase_transformation.py @@ -2,7 +2,9 @@ from unittest.mock import AsyncMock, Mock import httpx import pytest +import respx +import litellm from litellm.llms.predibase.chat.handler import PredibaseChatCompletion from litellm.llms.predibase.chat.transformation import PredibaseConfig from litellm.llms.predibase.common_utils import PredibaseError @@ -115,6 +117,24 @@ def test_predibase_get_complete_url_with_tenant_id_key(): assert url.endswith("/generate") +def test_predibase_authentication_error_does_not_expose_the_api_key(respx_mock: respx.MockRouter) -> None: + route = respx_mock.post( + "https://serving.app.predibase.com/c4768f95/deployments/v2/llms/llama-3-8b-instruct/generate" + ).mock(return_value=httpx.Response(401, json={"message": "invalid API key"})) + + with pytest.raises(litellm.AuthenticationError) as exc_info: + litellm.completion( + model="predibase/llama-3-8b-instruct", + messages=[{"role": "user", "content": "What is the meaning of life?"}], + predibase_tenant_id="c4768f95", + api_key="hf-rawapikey", + max_retries=0, + ) + + assert "hf-rawapikey" not in str(exc_info.value) + assert route.calls.last.request.headers["Authorization"] == "Bearer hf-rawapikey" + + def test_predibase_transform_response_success_best_of(monkeypatch): config = PredibaseConfig() logging_obj = Mock() diff --git a/tests/unit/llms/together_ai/test_together_ai_embedding.py b/tests/unit/llms/together_ai/test_together_ai_embedding.py new file mode 100644 index 00000000000..122f159b2d0 --- /dev/null +++ b/tests/unit/llms/together_ai/test_together_ai_embedding.py @@ -0,0 +1,36 @@ +import json +from typing import Final + +import httpx +import respx + +import litellm + + +def test_together_ai_embedding_uses_together_embeddings_endpoint(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://api.together.ai/v1/embeddings").mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.3, 0.4]}], + "model": "BAAI/bge-large-en-v1.5", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + }, + ) + ) + + response: Final = litellm.embedding( + model="together_ai/BAAI/bge-large-en-v1.5", + input=["hello"], + api_key="test-together-key", + ) + + assert str(route.calls.last.request.url) == "https://api.together.ai/v1/embeddings" + assert route.calls.last.request.headers["Authorization"] == "Bearer test-together-key" + assert json.loads(route.calls.last.request.read()) == { + "model": "BAAI/bge-large-en-v1.5", + "input": ["hello"], + } + assert response.data[0]["embedding"] == [0.3, 0.4] + assert (response.usage.prompt_tokens, response.usage.total_tokens) == (1, 1) diff --git a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_handler.py b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_handler.py index 483b350fd43..231bc25a7f0 100644 --- a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_handler.py +++ b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_handler.py @@ -1,13 +1,19 @@ +import json from typing import Final import httpx import pytest +import respx +import litellm from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_handler import GoogleBatchEmbeddings FILES_URI: Final = "https://generativelanguage.googleapis.com/v1beta/files/clip123" FILE_METADATA_URL: Final = "https://generativelanguage.googleapis.com/v1beta/files/clip123" +GEMINI_BATCH_EMBEDDINGS_URL: Final = ( + "https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-001:batchEmbedContents" +) def _files_api_metadata(request: httpx.Request) -> httpx.Response: @@ -26,6 +32,51 @@ def test_resolve_file_references_fetches_the_file_metadata_for_both_reference_fo assert resolved == {reference: {"mime_type": "video/mp4", "uri": FILES_URI}} +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize("embedding_input", ["good morning from litellm", ["good morning from litellm"]]) +@pytest.mark.asyncio +async def test_gemini_embedding_sends_batch_request_and_parses_vectors( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, + embedding_input: str | list[str], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post(GEMINI_BATCH_EMBEDDINGS_URL).mock( + return_value=httpx.Response( + 200, + json={"embeddings": [{"values": [0.1, 0.2, 0.3]}]}, + ) + ) + + response: Final = ( + litellm.embedding(model="gemini/gemini-embedding-001", input=embedding_input, api_key="gemini-test-key") + if sync_mode + else await litellm.aembedding( + model="gemini/gemini-embedding-001", input=embedding_input, api_key="gemini-test-key" + ) + ) + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content) == { + "requests": [ + { + "model": "models/gemini-embedding-001", + "content": {"parts": [{"text": "good morning from litellm"}]}, + } + ] + } + assert [item["embedding"] for item in response.data] == [[0.1, 0.2, 0.3]] + local_token_count: Final = litellm.token_counter( + model="gemini-embedding-001", text="good morning from litellm" + ) + assert local_token_count > 0 + assert (response.usage.prompt_tokens, response.usage.total_tokens) == ( + local_token_count, + local_token_count, + ) + + @pytest.mark.asyncio @pytest.mark.parametrize("reference", ["files/clip123", FILES_URI]) async def test_async_resolve_file_references_fetches_the_file_metadata_for_both_reference_forms( diff --git a/tests/unit/llms/vertex_ai/test_vertex.py b/tests/unit/llms/vertex_ai/test_vertex.py index 94dc7dc861d..f88c709ed68 100644 --- a/tests/unit/llms/vertex_ai/test_vertex.py +++ b/tests/unit/llms/vertex_ai/test_vertex.py @@ -1,7 +1,15 @@ import base64 +import functools import json +import re +import sys +from types import SimpleNamespace +from typing import Final from dotenv import load_dotenv +import httpx +import respx +from google.oauth2.credentials import Credentials import litellm.litellm_core_utils import litellm.litellm_core_utils.prompt_templates @@ -13,7 +21,7 @@ from unittest.mock import MagicMock import pytest import litellm -from litellm import get_optional_params +from litellm import Router, get_optional_params from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media from litellm.types.llms.vertex_ai import BlobType @@ -1535,3 +1543,1407 @@ def test_system_prompt_only_adds_blank_user_message(): ######################################################### assert len(data["system_instruction"]) == 1 assert data["system_instruction"]["parts"][0]["text"] == SYSTEM_INSTRUCTION + + +@pytest.fixture +def _cached_vertex_credentials(monkeypatch: pytest.MonkeyPatch) -> str: + from litellm.main import ( + vertex_chat_completion, + vertex_embedding, + vertex_model_garden_chat_completion, + vertex_partner_models_chat_completion, + ) + + credential_key: Final = "test-vertex-gemini-credentials" + credentials: Final = Credentials(token="test-vertex-token") + for vertex_configuration in ( + vertex_chat_completion, + vertex_embedding, + vertex_model_garden_chat_completion, + vertex_partner_models_chat_completion, + ): + monkeypatch.setitem( + vertex_configuration._credentials_project_mapping, + (credential_key, "test-project"), + (credentials, "test-project"), + ) + return credential_key + + +def test_litellm_completion_vertex_exception( + respx_mock: respx.MockRouter, _cached_vertex_credentials: str +) -> None: + url: Final = ( + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/" + "locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response( + 400, + json={ + "error": { + "code": 400, + "message": "Gemini rejected the request.", + "status": "INVALID_ARGUMENT", + } + }, + ) + ) + + with pytest.raises(litellm.BadRequestError) as exc_info: + litellm.completion( + model="vertex_ai/gemini-3.5-flash-lite", + messages=[{"role": "user", "content": "hello"}], + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=_cached_vertex_credentials, + ) + + assert exc_info.value.status_code == 400 + assert exc_info.value.llm_provider == "vertex_ai" + assert "Gemini rejected the request." in exc_info.value.message + assert route.call_count == 1 + + +def test_router_completion_vertex_exception( + respx_mock: respx.MockRouter, _cached_vertex_credentials: str +) -> None: + url: Final = ( + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/" + "locations/us-central1/publishers/google/models/gemini-3.5-flash-lite:generateContent" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response( + 429, + json={ + "error": { + "code": 429, + "message": "Vertex Gemini quota exceeded.", + "status": "RESOURCE_EXHAUSTED", + } + }, + ) + ) + router: Final = Router( + model_list=[ + { + "model_name": "vertex-gemini-test", + "litellm_params": { + "model": "vertex_ai/gemini-3.5-flash-lite", + "vertex_project": "test-project", + "vertex_location": "us-central1", + "vertex_credentials": _cached_vertex_credentials, + }, + } + ], + num_retries=0, + ) + + with pytest.raises(litellm.RateLimitError) as exc_info: + router.completion( + model="vertex-gemini-test", + messages=[{"role": "user", "content": "hello"}], + ) + + assert exc_info.value.status_code == 429 + assert exc_info.value.llm_provider == "vertex_ai" + assert "Vertex Gemini quota exceeded." in exc_info.value.message + assert route.call_count == 1 + + +def _vertex_gemini_response(text: str = "hello") -> dict[str, object]: + return { + "candidates": [ + { + "content": {"role": "model", "parts": [{"text": text}]}, + "finishReason": "STOP", + } + ], + "usageMetadata": { + "promptTokenCount": 1, + "candidatesTokenCount": 1, + "totalTokenCount": 2, + }, + } + + +@pytest.mark.asyncio +async def test_completion_fine_tuned_model( + respx_mock: respx.MockRouter, + _cached_vertex_credentials: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + url: Final = ( + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/" + "locations/us-central1/endpoints/4965075652664360960:generateContent" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response(200, json=_vertex_gemini_response("A canvas vast")) + ) + + response: Final = await litellm.acompletion( + model="vertex_ai_beta/4965075652664360960", + messages=[{"role": "user", "content": "Write a short poem about the sky"}], + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=_cached_vertex_credentials, + max_retries=0, + ) + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content) == { + "contents": [ + {"role": "user", "parts": [{"text": "Write a short poem about the sky"}]} + ] + } + assert response.choices[0].message.content.startswith("A canvas vast") + assert response.choices[0].finish_reason == "stop" + assert response.usage.total_tokens == 2 + + +@pytest.mark.parametrize( + ("base_model", "metadata"), + [ + (None, {"model_info": {"base_model": "vertex_ai/gemini-1.5-pro"}}), + ("vertex_ai/gemini-1.5-pro", None), + ], +) +def test_gemini_finetuned_endpoint( + respx_mock: respx.MockRouter, + _cached_vertex_credentials: str, + base_model: str | None, + metadata: dict[str, object] | None, +) -> None: + url: Final = ( + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/" + "locations/us-central1/endpoints/4965075652664360960:generateContent" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response(200, json=_vertex_gemini_response()) + ) + + litellm.completion( + model="vertex_ai/4965075652664360960", + messages=[{"role": "user", "content": "search for weather in boston"}], + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=_cached_vertex_credentials, + base_model=base_model, + metadata=metadata, + max_retries=0, + ) + + assert route.call_count == 1 + assert str(route.calls[0].request.url).endswith( + "endpoints/4965075652664360960:generateContent" + ) + + +def test_gemini_nullable_object_tool_schema_httpx( + respx_mock: respx.MockRouter, _cached_vertex_credentials: str +) -> None: + url: Final = ( + "https://aiplatform.googleapis.com/v1/projects/test-project/locations/global/" + "publishers/google/models/gemini-3.5-flash:generateContent" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response(200, json=_vertex_gemini_response()) + ) + + litellm.completion( + model="vertex_ai/gemini-3.5-flash", + messages=[{"role": "user", "content": "call the tool"}], + tools=[ + { + "type": "function", + "strict": True, + "function": { + "name": "create_support_ticket", + "description": "Create a support ticket", + "parameters": { + "type": "object", + "additionalProperties": False, + "required": ["ticket_id", "customer_context"], + "properties": { + "ticket_id": {"type": "string"}, + "optional_context": { + "type": ["object", "null"], + "additionalProperties": False, + "required": ["source"], + "properties": {"source": {"type": "string"}}, + }, + "customer_context": { + "anyOf": [ + { + "type": "object", + "additionalProperties": False, + "required": ["user_id", "plan"], + "properties": { + "user_id": {"type": "string"}, + "plan": {"type": "string"}, + }, + }, + {"type": "null"}, + ], + }, + }, + }, + }, + } + ], + tool_choice="required", + vertex_project="test-project", + vertex_location="global", + vertex_credentials=_cached_vertex_credentials, + max_retries=0, + ) + + assert route.call_count == 1 + request_body: Final = json.loads(route.calls[0].request.content) + parameters_schema: Final = request_body["tools"][0]["function_declarations"][0][ + "parameters" + ] + context_schema: Final = parameters_schema["properties"]["customer_context"] + nullable_object_schema: Final = context_schema["anyOf"][0] + assert nullable_object_schema["nullable"] is True + assert nullable_object_schema["properties"] == { + "user_id": {"type": "string"}, + "plan": {"type": "string"}, + } + assert parameters_schema["properties"]["optional_context"]["anyOf"] == [ + { + "type": "object", + "required": ["source"], + "properties": {"source": {"type": "string"}}, + }, + {"type": "null"}, + ] + + +@pytest.mark.parametrize("value_in_dict", [{}, {"disable_attribution": False}]) +def test_gemini_pro_grounding( + respx_mock: respx.MockRouter, + _cached_vertex_credentials: str, + value_in_dict: dict[str, bool], +) -> None: + url: Final = ( + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/" + "locations/us-central1/publishers/google/models/gemini-1.0-pro-001:generateContent" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response( + 200, + json={ + "candidates": [ + { + "content": { + "role": "model", + "parts": [{"text": "Argentina won the World Cup"}], + }, + "finishReason": "STOP", + "groundingMetadata": { + "webSearchQueries": ["latest world cup winner"], + "groundingChunks": [ + { + "web": { + "uri": "https://example.com", + "title": "Example", + } + } + ], + }, + } + ], + "usageMetadata": { + "promptTokenCount": 1, + "candidatesTokenCount": 2, + "totalTokenCount": 3, + }, + }, + ) + ) + + response: Final = litellm.completion( + model="vertex_ai_beta/gemini-1.0-pro-001", + messages=[{"role": "user", "content": "Who won the world cup?"}], + tools=[{"googleSearchRetrieval": value_in_dict}], + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=_cached_vertex_credentials, + max_retries=0, + ) + + assert route.call_count == 1 + request_body: Final = json.loads(route.calls[0].request.content) + assert request_body["tools"] == [{"googleSearchRetrieval": value_in_dict}] + assert response._hidden_params["vertex_ai_grounding_metadata"] == [ + { + "webSearchQueries": ["latest world cup winner"], + "groundingChunks": [ + {"web": {"uri": "https://example.com", "title": "Example"}} + ], + } + ] + + +_CUSTOM_API_BASE: Final = "https://vertex-proxy.example.com/v1/custom-route" + + +@pytest.mark.parametrize( + ("model", "endpoint", "response_body"), + [ + ("gemini-2.5-flash-lite", "generateContent", _vertex_gemini_response()), + ( + "claude-3-5-sonnet@20240620", + "rawPredict", + { + "id": "msg-custom-base", + "type": "message", + "role": "assistant", + "model": "claude-3-5-sonnet-20240620", + "content": [{"type": "text", "text": "hello"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ), + ], +) +def test_gemini_pro_httpx_custom_api_base( + respx_mock: respx.MockRouter, + _cached_vertex_credentials: str, + model: str, + endpoint: str, + response_body: dict[str, object], +) -> None: + route: Final = respx_mock.post(f"{_CUSTOM_API_BASE}:{endpoint}").mock( + return_value=httpx.Response(200, json=response_body) + ) + + response: Final = litellm.completion( + model=f"vertex_ai/{model}", + messages=[{"role": "user", "content": "Hello world"}], + api_base=_CUSTOM_API_BASE, + extra_headers={"hello": "world"}, + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=_cached_vertex_credentials, + max_retries=0, + ) + + assert route.call_count == 1 + assert str(route.calls[0].request.url) == f"{_CUSTOM_API_BASE}:{endpoint}" + assert route.calls[0].request.headers["hello"] == "world" + assert response.choices[0].message.content == "hello" + + +def test_prompt_factory() -> None: + from litellm.llms.vertex_ai.gemini.transformation import ( + gemini_convert_messages_with_history, + ) + + messages: Final = [ + { + "role": "system", + "content": "You are a helpful assistant.", + }, + {"role": "user", "content": "What is the weather in San Francisco?"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_weather", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location":"San Francisco"}', + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_weather", + "content": "Sunny, 68 degrees", + }, + ] + + translated: Final = gemini_convert_messages_with_history(messages=messages) + + assert translated == [ + { + "role": "user", + "parts": [ + {"text": "You are a helpful assistant."}, + {"text": "What is the weather in San Francisco?"}, + ], + }, + { + "role": "model", + "parts": [ + {"text": ""}, + { + "function_call": { + "name": "get_weather", + "args": {"location": "San Francisco"}, + } + }, + ], + }, + { + "role": "user", + "parts": [ + { + "function_response": { + "name": "get_weather", + "response": {"content": "Sunny, 68 degrees"}, + } + } + ], + }, + ] + + +def test_signed_s3_url_with_format( + respx_mock: respx.MockRouter, _cached_vertex_credentials: str +) -> None: + url: Final = ( + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/" + "locations/us-central1/publishers/google/models/gemini-2.0-flash-001:generateContent" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response(200, json=_vertex_gemini_response()) + ) + + litellm.completion( + model="vertex_ai/gemini-2.0-flash-001", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://images.example.com/logo.png?signature=test", + "format": "image/jpeg", + }, + }, + {"type": "text", "text": "Describe this image"}, + ], + } + ], + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=_cached_vertex_credentials, + max_retries=0, + ) + + assert route.call_count == 1 + request_body: Final = json.loads(route.calls[0].request.content) + assert request_body["contents"][0]["parts"] == [ + { + "file_data": { + "mime_type": "image/jpeg", + "file_uri": "https://images.example.com/logo.png?signature=test", + } + }, + {"text": "Describe this image"}, + ] + + +@pytest.mark.asyncio +async def test_vertex_ai_deepseek( + respx_mock: respx.MockRouter, + _cached_vertex_credentials: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post( + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/" + "locations/us-central1/endpoints/openapi/chat/completions" + ).mock( + return_value=httpx.Response( + 200, + json={ + "choices": [ + { + "message": {"role": "assistant", "content": "Hello"}, + "index": 0, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + "model": "deepseek-ai/deepseek-r1-0528-maas", + }, + ) + ) + + await litellm.acompletion( + model="vertex_ai/deepseek-ai/deepseek-r1-0528-maas", + messages=[{"role": "user", "content": "Hi!"}], + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=_cached_vertex_credentials, + max_retries=0, + ) + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content) == { + "model": "deepseek-ai/deepseek-r1-0528-maas", + "messages": [{"role": "user", "content": "Hi!"}], + "stream": False, + } + + +def test_vertex_ai_response_id( + respx_mock: respx.MockRouter, _cached_vertex_credentials: str +) -> None: + url: Final = ( + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/" + "locations/us-central1/publishers/google/models/gemini-1.5-pro:generateContent" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response( + 200, + json={ + "responseId": "vertex_ai_response_123", + "candidates": [ + { + "content": { + "role": "model", + "parts": [{"text": "Hello! How can I help you today?"}], + }, + "finishReason": "STOP", + } + ], + "usageMetadata": { + "promptTokenCount": 10, + "candidatesTokenCount": 8, + "totalTokenCount": 18, + }, + }, + ) + ) + + response: Final = litellm.completion( + model="vertex_ai/gemini-1.5-pro", + messages=[{"role": "user", "content": "Hi!"}], + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=_cached_vertex_credentials, + max_retries=0, + ) + + assert route.call_count == 1 + assert response.id == "vertex_ai_response_123" + assert response.choices[0].message.content == "Hello! How can I help you today?" + + +@pytest.mark.asyncio +async def test_vertexai_embedding_finetuned( + respx_mock: respx.MockRouter, + _cached_vertex_credentials: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + url: Final = ( + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/" + "locations/us-central1/endpoints/1004708436694269952:predict" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response( + 200, + json={ + "predictions": [ + [[-0.000431762, -0.04416759, -0.03443353]], + [[-0.000431762, -0.04416759, -0.03443353]], + ] + }, + ) + ) + inputs: Final = ["good morning from litellm", "this is another item"] + + response: Final = await litellm.aembedding( + model="vertex_ai/1004708436694269952", + input=inputs, + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=_cached_vertex_credentials, + max_retries=0, + ) + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content) == { + "instances": [{"inputs": item} for item in inputs], + "parameters": {}, + } + assert [item["embedding"] for item in response.data] == [ + [-0.000431762, -0.04416759, -0.03443353], + [-0.000431762, -0.04416759, -0.03443353], + ] + + +@pytest.mark.parametrize("max_retries", [None, 3]) +@pytest.mark.asyncio +async def test_vertexai_model_garden_model_completion( + respx_mock: respx.MockRouter, + _cached_vertex_credentials: str, + monkeypatch: pytest.MonkeyPatch, + max_retries: int | None, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setitem( + sys.modules, + "vertexai", + SimpleNamespace(preview=SimpleNamespace(language_models=SimpleNamespace())), + ) + url: Final = ( + "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/test-project/" + "locations/us-central1/endpoints/5464397967697903616/chat/completions" + ) + messages: Final = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"}, + ] + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response( + 200, + json={ + "id": "chat-vertex-model-garden", + "object": "chat.completion", + "created": 1731702782, + "model": "meta-llama/Llama-3.1-8B-Instruct", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Litellm Bot"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 3, + "completion_tokens": 2, + "total_tokens": 5, + }, + }, + ) + ) + + response: Final = await litellm.acompletion( + model="vertex_ai/openai/5464397967697903616", + messages=messages, + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=_cached_vertex_credentials, + max_retries=max_retries, + ) + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content) == { + "model": "", + "messages": messages, + "stream": False, + } + assert response.id == "chat-vertex-model-garden" + assert response.created == 1731702782 + assert response.model == "vertex_ai/meta-llama/Llama-3.1-8B-Instruct" + assert len(response.choices) == 1 + assert response.choices[0].message.role == "assistant" + assert response.choices[0].message.content == "Litellm Bot" + assert response.choices[0].finish_reason == "stop" + assert ( + response.usage.prompt_tokens, + response.usage.completion_tokens, + response.usage.total_tokens, + ) == (3, 2, 5) + + +@pytest.mark.asyncio +async def test_partner_models_httpx_ai21( + respx_mock: respx.MockRouter, + _cached_vertex_credentials: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + url: Final = ( + "https://us-central1-aiplatform.googleapis.com/v1beta1/projects/test-project/" + "locations/us-central1/publishers/ai21/models/jamba-1.5-mini@001:rawPredict" + ) + messages: Final = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is the weather in San Francisco?"}, + ] + tools: Final = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, + } + ] + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response( + 200, + json={ + "id": "chat-ai21", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Checking the weather.", + "tool_calls": [ + { + "id": "call-weather", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location":"San Francisco"}', + }, + } + ], + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 3, + "completion_tokens": 4, + "total_tokens": 7, + }, + "model": "jamba-1.5-mini@001", + }, + ) + ) + + response: Final = await litellm.acompletion( + model="vertex_ai/jamba-1.5-mini@001", + messages=messages, + tools=tools, + top_p=0.5, + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=_cached_vertex_credentials, + max_retries=0, + ) + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content) == { + "model": "jamba-1.5-mini@001", + "messages": messages, + "top_p": 0.5, + "tools": tools, + "stream": False, + } + assert response.id == "chat-ai21" + assert len(response.choices) == 1 + assert response.choices[0].message.content == "Checking the weather." + assert response.choices[0].message.tool_calls[0].function.name == "get_weather" + assert ( + response.choices[0].message.tool_calls[0].function.arguments + == '{"location":"San Francisco"}' + ) + assert ( + response.usage.prompt_tokens, + response.usage.completion_tokens, + response.usage.total_tokens, + ) == (3, 4, 7) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_gemini_context_caching_anthropic_format( + respx_mock: respx.MockRouter, + fake_provider_credentials: None, + sync_mode: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + cache_list_route: Final = respx_mock.get( + url__regex=r".*cachedContents.*" + ).mock(return_value=httpx.Response(200, json={"cachedContents": []})) + cache_route: Final = respx_mock.post(url__regex=r".*cachedContents.*").mock( + return_value=httpx.Response( + 200, + json={ + "name": "cachedContents/test-cache", + "model": "models/gemini-2.5-flash-lite-001", + "expireTime": "2024-08-26T22:36:15Z", + "usageMetadata": {"totalTokenCount": 323383}, + }, + ) + ) + generation_route: Final = respx_mock.post( + url__regex=r".*generateContent.*" + ).mock(return_value=httpx.Response(200, json=_vertex_gemini_response())) + messages: Final = [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Legal agreement " * 4000, + "cache_control": {"type": "ephemeral"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Summarize the agreement", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + {"role": "assistant", "content": "The agreement lasts one year."}, + {"role": "user", "content": "What is its duration?"}, + ] + + if sync_mode: + litellm.completion( + model="gemini/gemini-2.5-flash-lite-001", + messages=messages, + temperature=0.2, + max_tokens=10, + ) + else: + await litellm.acompletion( + model="gemini/gemini-2.5-flash-lite-001", + messages=messages, + temperature=0.2, + max_tokens=10, + ) + + assert cache_route.call_count == 1 + assert generation_route.call_count == 1 + assert cache_list_route.call_count == 1 + assert [(call.request.method, str(call.request.url)) for call in respx_mock.calls] == [ + ("GET", "https://generativelanguage.googleapis.com/v1beta/cachedContents"), + ("POST", "https://generativelanguage.googleapis.com/v1beta/cachedContents"), + ( + "POST", + "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-lite-001:generateContent", + ), + ] + cache_request: Final = json.loads(cache_route.calls[0].request.content) + assert cache_request["model"] == "models/gemini-2.5-flash-lite-001" + assert cache_request["contents"] == [ + {"role": "user", "parts": [{"text": "Summarize the agreement"}]} + ] + assert cache_request["system_instruction"] == { + "parts": [{"text": "Legal agreement " * 4000}] + } + assert json.loads(generation_route.calls[0].request.content) == { + "contents": [ + {"role": "model", "parts": [{"text": "The agreement lasts one year."}]}, + {"role": "user", "parts": [{"text": "What is its duration?"}]}, + ], + "generationConfig": {"temperature": 0.2, "max_output_tokens": 10}, + "cachedContent": "cachedContents/test-cache", + } + + +@pytest.mark.asyncio +async def test_anthropic_message_via_anthropic_messages( + respx_mock: respx.MockRouter, + _cached_vertex_credentials: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( + VertexAIPartnerModelsAnthropicMessagesConfig, + ) + from litellm.utils import ProviderConfigManager + + provider_configs: Final = ( + ProviderConfigManager.get_provider_anthropic_messages_config( + model="vertex_ai/claude-3-5-sonnet@20240620", + provider=litellm.LlmProviders.VERTEX_AI, + ), + ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude-3-5-sonnet@20240620", + provider=litellm.LlmProviders.VERTEX_AI, + ), + ) + for provider_config in provider_configs: + assert isinstance(provider_config, VertexAIPartnerModelsAnthropicMessagesConfig) + monkeypatch.setitem( + provider_config._credentials_project_mapping, + (_cached_vertex_credentials, "test-project"), + (Credentials(token="test-vertex-token"), "test-project"), + ) + route: Final = respx_mock.post( + url__regex=r".*claude-3-5-sonnet@20240620.*" + ).mock( + return_value=httpx.Response( + 200, + json={ + "id": "msg-vertex", + "type": "message", + "role": "assistant", + "model": "claude-3-5-sonnet-20240620", + "content": [{"type": "text", "text": "Here are the recipes."}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 4, "output_tokens": 5}, + }, + ) + ) + messages: Final = [{"role": "user", "content": "List 5 cookie recipes"}] + call_params: Final = { + "model": "vertex_ai/claude-3-5-sonnet@20240620", + "messages": messages, + "max_tokens": 100, + "vertex_project": "test-project", + "vertex_location": "us-east5", + "vertex_credentials": _cached_vertex_credentials, + "max_retries": 0, + } + + response: Final = await litellm.anthropic_messages(**call_params) + completion_response: Final = await litellm.acompletion(**call_params) + + assert response["stop_reason"] == "end_turn" + assert completion_response.choices[0].message.content == "Here are the recipes." + assert route.call_count == 2 + messages_request: Final = route.calls[0].request + completion_request: Final = route.calls[1].request + assert str(messages_request.url) == ( + "https://us-east5-aiplatform.googleapis.com/v1/projects/test-project/locations/us-east5/" + "publishers/anthropic/models/claude-3-5-sonnet@20240620:rawPredict" + ) + assert str(completion_request.url) == str(messages_request.url) + for request in (messages_request, completion_request): + assert request.headers["authorization"] == "Bearer test-vertex-token" + assert request.headers["content-type"] == "application/json" + messages_body: Final = json.loads(messages_request.content) + assert messages_body == { + "anthropic_version": "vertex-2023-10-16", + "max_tokens": 100, + "messages": messages, + "stream": False, + } + assert set(json.loads(completion_request.content)) <= set(messages_body) + + +def test_gemini_fine_tuned_model_request_consistency( + respx_mock: respx.MockRouter, _cached_vertex_credentials: str +) -> None: + route: Final = respx_mock.post( + url__regex=r".*publishers/google/models/.*:generateContent" + ).mock(return_value=httpx.Response(200, json=_vertex_gemini_response())) + messages: Final = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is the weather?"}, + ] + tools: Final = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, + } + ] + common_params: Final = { + "messages": messages, + "tools": tools, + "tool_choice": "auto", + "vertex_project": "test-project", + "vertex_location": "us-central1", + "vertex_credentials": _cached_vertex_credentials, + "max_retries": 0, + } + + litellm.completion(model="vertex_ai/gemini/ft-uuid", **common_params) + litellm.completion(model="vertex_ai/gemini-2.5-flash", **common_params) + + assert route.call_count == 2 + first_request: Final = json.loads(route.calls[0].request.content) + second_request: Final = json.loads(route.calls[1].request.content) + assert str(route.calls[0].request.url) == ( + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/" + "publishers/google/models/ft-uuid:generateContent" + ) + assert first_request["system_instruction"] == { + "parts": [{"text": "You are a helpful assistant."}] + } + assert first_request["tools"][0]["function_declarations"][0]["name"] == "get_weather" + assert first_request == second_request + + +_RECIPES_SCHEMA: Final = { + "type": "object", + "properties": { + "recipes": { + "type": "array", + "items": { + "type": "object", + "properties": {"recipe_name": {"type": "string"}}, + "required": ["recipe_name"], + }, + } + }, + "required": ["recipes"], + "additionalProperties": False, +} + + +def _recipes_response(model: str, invalid_response: bool) -> dict[str, object]: + if "claude" in model and invalid_response: + return { + "id": "msg-recipes", + "type": "message", + "role": "assistant", + "model": "claude-3-5-sonnet-20240620", + "content": [{"type": "text", "text": "Hi! My name is Claude."}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 4}, + } + if "claude" in model: + return { + "id": "msg-recipes", + "type": "message", + "role": "assistant", + "model": "claude-3-5-sonnet-20240620", + "content": [ + { + "type": "tool_use", + "id": "toolu-recipes", + "name": "json_tool_call", + "input": {"recipes": [{"recipe_name": "Sugar Cookies"}]}, + } + ], + "stop_reason": "tool_use", + "usage": {"input_tokens": 3, "output_tokens": 4}, + } + if invalid_response: + return _vertex_gemini_response('[{"recipe_world": "Sugar Cookies"}]') + return _vertex_gemini_response('{"recipes": [{"recipe_name": "Sugar Cookies"}]}') + + +_JSON_SCHEMA_MODELS: Final = pytest.mark.parametrize( + ("model", "vertex_location", "endpoint"), + [ + ("vertex_ai_beta/gemini-2.0-flash-001", "us-central1", "gemini-2.0-flash-001:generateContent"), + ("vertex_ai_beta/gemini-2.5-flash-lite", "us-central1", "gemini-2.5-flash-lite:generateContent"), + ("vertex_ai/claude-3-5-sonnet@20240620", "us-east5", "claude-3-5-sonnet@20240620:rawPredict"), + ], +) + + +@_JSON_SCHEMA_MODELS +@pytest.mark.parametrize("invalid_response", [True, False]) +@pytest.mark.parametrize("enforce_validation", [True, False]) +def test_gemini_pro_json_schema_args_sent_httpx( + respx_mock: respx.MockRouter, + _cached_vertex_credentials: str, + model: str, + vertex_location: str, + endpoint: str, + invalid_response: bool, + enforce_validation: bool, +) -> None: + route: Final = respx_mock.post(url__regex=rf".*/{re.escape(endpoint)}$").mock( + return_value=httpx.Response(200, json=_recipes_response(model, invalid_response)) + ) + call: Final = functools.partial( + litellm.completion, + model=model, + messages=[{"role": "user", "content": "List 5 cookie recipes"}], + response_format={ + "type": "json_object", + "response_schema": _RECIPES_SCHEMA, + "enforce_validation": enforce_validation, + }, + vertex_project="test-project", + vertex_location=vertex_location, + vertex_credentials=_cached_vertex_credentials, + max_retries=0, + ) + + if invalid_response and enforce_validation: + with pytest.raises(litellm.JSONSchemaValidationError): + call() + else: + response = call() + assert response.model == model.split("/")[1] + if not invalid_response: + assert json.loads(response.choices[0].message.content) == { + "recipes": [{"recipe_name": "Sugar Cookies"}] + } + + assert route.call_count == 1 + request_body: Final = json.loads(route.calls[0].request.content) + if "claude" in model: + assert request_body["tool_choice"] == {"type": "tool", "name": "json_tool_call"} + assert request_body["tools"] == [{"name": "json_tool_call", "input_schema": _RECIPES_SCHEMA}] + else: + assert request_body["generationConfig"]["response_mime_type"] == "application/json" + assert request_body["generationConfig"]["response_json_schema"] == _RECIPES_SCHEMA + + +@_JSON_SCHEMA_MODELS +@pytest.mark.parametrize("invalid_response", [True, False]) +@pytest.mark.parametrize("enforce_validation", [True, False]) +def test_gemini_pro_json_schema_args_sent_httpx_openai_schema( + respx_mock: respx.MockRouter, + _cached_vertex_credentials: str, + monkeypatch: pytest.MonkeyPatch, + model: str, + vertex_location: str, + endpoint: str, + invalid_response: bool, + enforce_validation: bool, +) -> None: + from pydantic import BaseModel + + class Recipe(BaseModel): + recipe_name: str + + class ResponseSchema(BaseModel): + recipes: list[Recipe] + + monkeypatch.setattr(litellm, "enable_json_schema_validation", enforce_validation) + route: Final = respx_mock.post(url__regex=rf".*/{re.escape(endpoint)}$").mock( + return_value=httpx.Response(200, json=_recipes_response(model, invalid_response)) + ) + call: Final = functools.partial( + litellm.completion, + model=model, + messages=[{"role": "user", "content": "List 5 cookie recipes"}], + response_format=ResponseSchema, + vertex_project="test-project", + vertex_location=vertex_location, + vertex_credentials=_cached_vertex_credentials, + max_retries=0, + ) + + if invalid_response and enforce_validation: + with pytest.raises(litellm.JSONSchemaValidationError): + call() + else: + response = call() + assert response.model == model.split("/")[1] + if not invalid_response: + assert json.loads(response.choices[0].message.content) == { + "recipes": [{"recipe_name": "Sugar Cookies"}] + } + + assert route.call_count == 1 + request_body: Final = json.loads(route.calls[0].request.content) + if "claude" in model: + assert request_body["tool_choice"] == {"type": "tool", "name": "json_tool_call"} + assert set(request_body["tools"][0]["input_schema"]["properties"]) == {"recipes"} + else: + generation_config: Final = request_body["generationConfig"] + assert generation_config["response_mime_type"] == "application/json" + assert set(generation_config["response_json_schema"]["properties"]) == {"recipes"} + + +@pytest.mark.parametrize("content_filter_type", ["prompt", "response"]) +@pytest.mark.asyncio +async def test_gemini_pro_json_schema_httpx_content_policy_error( + respx_mock: respx.MockRouter, + _cached_vertex_credentials: str, + content_filter_type: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + response_body: Final = ( + { + "promptFeedback": {"blockReason": "OTHER"}, + "usageMetadata": {"promptTokenCount": 1, "totalTokenCount": 1}, + } + if content_filter_type == "prompt" + else { + "candidates": [ + { + "content": {"role": "model", "parts": []}, + "finishReason": "SAFETY", + } + ], + "usageMetadata": {"promptTokenCount": 1, "totalTokenCount": 1}, + } + ) + route: Final = respx_mock.post( + url__regex=r".*gemini-2.5-flash-lite:generateContent" + ).mock(return_value=httpx.Response(200, json=response_body)) + + response: Final = await litellm.acompletion( + model="vertex_ai_beta/gemini-2.5-flash-lite", + messages=[{"role": "user", "content": "List recipes"}], + response_format={"type": "json_object"}, + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=_cached_vertex_credentials, + max_retries=0, + ) + + assert route.call_count == 1 + assert response.choices[0].finish_reason == "content_filter" + + +@pytest.mark.parametrize("route_type", ["completion", "embedding"]) +def test_litellm_api_base( + respx_mock: respx.MockRouter, + _cached_vertex_credentials: str, + monkeypatch: pytest.MonkeyPatch, + route_type: str, +) -> None: + monkeypatch.setattr(litellm, "api_base", "https://litellm.com") + route: Final = respx_mock.post(url__regex=r"https://litellm\.com.*").mock( + return_value=httpx.Response( + 200, + json=( + _vertex_gemini_response() + if route_type == "completion" + else { + "predictions": [ + {"embeddings": {"values": [0.1, 0.2], "statistics": {"token_count": 1}}} + ] + } + ), + ) + ) + + if route_type == "completion": + litellm.completion( + model="vertex_ai/gemini-2.0-flash-001", + messages=[{"role": "user", "content": "Hello"}], + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=_cached_vertex_credentials, + max_retries=0, + ) + else: + litellm.embedding( + model="vertex_ai/gemini-2.0-flash-001", + input=["Hello"], + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=_cached_vertex_credentials, + max_retries=0, + ) + + assert route.call_count == 1 + assert str(route.calls[0].request.url) == ( + "https://litellm.com/v1/projects/test-project/locations/us-central1/publishers/google/models/" + f"gemini-2.0-flash-001:{'generateContent' if route_type == 'completion' else 'predict'}" + ) + + +def test_vertex_anthropic_completion( + respx_mock: respx.MockRouter, _cached_vertex_credentials: str +) -> None: + route: Final = respx_mock.post( + url__regex=r".*claude-sonnet-4-6@default.*" + ).mock( + return_value=httpx.Response( + 200, + json={ + "id": "msg-vertex", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6@default", + "content": [ + { + "type": "thinking", + "thinking": "I will greet the user.", + "signature": "test-signature", + }, + {"type": "text", "text": "Hello, world!"}, + ], + "stop_reason": "end_turn", + "usage": {"input_tokens": 2, "output_tokens": 3}, + }, + ) + ) + + response: Final = litellm.completion( + model="vertex_ai/claude-sonnet-4-6@default", + messages=[{"role": "user", "content": "Hello, world!"}], + vertex_project="test-project", + vertex_location="global", + vertex_credentials=_cached_vertex_credentials, + thinking={"type": "enabled", "budget_tokens": 1024}, + input_cost_per_token=0.001, + output_cost_per_token=0.002, + max_retries=0, + ) + + assert route.call_count == 1 + assert response.model == "claude-sonnet-4-6@default" + assert response._hidden_params["response_cost"] == pytest.approx( + 2 * 0.001 + 3 * 0.002 + ) + assert response.choices[0].message.reasoning_content == "I will greet the user." + assert response.choices[0].message.thinking_blocks == [ + { + "type": "thinking", + "thinking": "I will greet the user.", + "signature": "test-signature", + } + ] + assert response.choices[0].message.content == "Hello, world!" + + +def test_vertex_gemini_streaming_returns_text_and_finish_reason( + respx_mock: respx.MockRouter, _cached_vertex_credentials: str +) -> None: + route: Final = respx_mock.post( + url__regex=( + r"https://us-central1-aiplatform\.googleapis\.com/v1(?:beta1)?/projects/" + r"test-project/locations/us-central1/publishers/google/models/" + r"gemini-2\.5-flash-lite:streamGenerateContent.*" + ) + ).mock( + return_value=httpx.Response( + 200, + text=( + 'data: {"candidates":[{"content":{"parts":[{"text":"Vertex streams "}],' + '"role":"model"},"index":0}]}\n\n' + 'data: {"candidates":[{"content":{"parts":[{"text":"without a live API."}],' + '"role":"model"},"index":0,"finishReason":"STOP"}]}\n\n' + ), + headers={"content-type": "text/event-stream"}, + ) + ) + + stream: Final = litellm.completion( + model="vertex_ai_beta/gemini-2.5-flash-lite", + messages=[{"role": "user", "content": "Say a sentence."}], + stream=True, + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=_cached_vertex_credentials, + max_retries=0, + ) + chunks: Final = tuple(stream) + + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == ( + "Vertex streams without a live API." + ) + assert tuple( + chunk.choices[0].finish_reason + for chunk in chunks + if chunk.choices[0].finish_reason is not None + ) == ("stop",) + assert route.call_count == 1 diff --git a/tests/unit/llms/vertex_ai/test_vertex_llm_base.py b/tests/unit/llms/vertex_ai/test_vertex_llm_base.py index b5a24ab02d9..514728137cf 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_llm_base.py +++ b/tests/unit/llms/vertex_ai/test_vertex_llm_base.py @@ -1,7 +1,10 @@ import asyncio import json +import logging +from typing import Final from unittest.mock import MagicMock, call, patch +import google.auth.exceptions import pytest from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH @@ -2180,3 +2183,19 @@ class TestVertexBase: assert token == "cached-token" assert not mock_get_lock.called, "Fast path should not acquire lock" + + +def test_get_access_token_reraises_malformed_service_account_credentials_without_caching(caplog): + vertex_base: Final = VertexBase() + + with caplog.at_level(logging.ERROR, logger="LiteLLM"): + with pytest.raises(google.auth.exceptions.MalformedError): + vertex_base.get_access_token( + credentials=json.dumps({"type": "service_account", "project_id": "project-1"}), + project_id="project-1", + ) + + assert vertex_base._credentials_project_mapping == {} + assert [record.getMessage().split(". Error: ")[0] for record in caplog.records] == [ + "Failed to load vertex credentials. Check to see if credentials containing partial/invalid information" + ] diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 0208d87b354..6cfbc0429bd 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -3,8 +3,12 @@ import copy import json from typing import Final +import httpx import pytest +import respx +from google.oauth2.credentials import Credentials +import litellm from litellm.anthropic_beta_headers_manager import ( update_headers_with_filtered_beta, update_request_with_filtered_beta, @@ -489,6 +493,64 @@ def test_vertex_ai_anthropic_no_extra_headers_unchanged(): assert "anthropic-beta" not in headers +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_vertex_ai_anthropic_prompt_caching_does_not_add_beta_header( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, +) -> None: + from litellm.main import vertex_partner_models_chat_completion + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setitem( + vertex_partner_models_chat_completion._credentials_project_mapping, + ("test-vertex-credentials", "test-project"), + (Credentials(token="test-vertex-token"), "test-project"), + ) + route: Final = respx_mock.post( + "https://us-east5-aiplatform.googleapis.com/v1/projects/test-project/locations/us-east5/" + "publishers/anthropic/models/claude-3-5-sonnet-v2@20241022:rawPredict" + ).mock( + return_value=httpx.Response( + 200, + json={ + "id": "msg-cache", + "type": "message", + "role": "assistant", + "model": "claude-3-5-sonnet-v2-20241022", + "content": [{"type": "text", "text": "cached"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 4, "output_tokens": 1}, + }, + ) + ) + cached_block: Final = { + "type": "text", + "text": "Cache this request", + "cache_control": {"type": "ephemeral"}, + } + call_params: Final = { + "model": "vertex_ai/claude-3-5-sonnet-v2@20241022", + "messages": [{"role": "user", "content": [cached_block]}], + "vertex_project": "test-project", + "vertex_location": "us-east5", + "vertex_credentials": "test-vertex-credentials", + "max_retries": 0, + } + + if sync_mode: + litellm.completion(**call_params) + else: + await litellm.acompletion(**call_params) + + assert route.call_count == 1 + request: Final = route.calls[0].request + assert json.loads(request.content)["messages"] == [{"role": "user", "content": [cached_block]}] + assert "anthropic-beta" not in request.headers + assert "anthropic_beta" not in json.loads(request.content) + + def test_vertex_ai_partner_models_anthropic_remove_prompt_caching_scope_beta_header(): """ Test that remove_unsupported_beta correctly filters out prompt-caching-scope-2026-01-05 diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/test_partner_models_credential_reuse.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/test_partner_models_credential_reuse.py index 8e6270e41a1..b0a0a8d33c1 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/test_partner_models_credential_reuse.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/test_partner_models_credential_reuse.py @@ -3,10 +3,17 @@ Test that VertexBase subclasses (PartnerModels, Gemma, ModelGarden) reuse cached credentials instead of creating a new VertexLLM instance on every request. """ +import json import sys +from typing import Final from unittest.mock import MagicMock, patch +import httpx import pytest +import responses +import respx + +import litellm from litellm.llms.vertex_ai.vertex_ai_partner_models.main import ( VertexAIPartnerModels, @@ -256,3 +263,60 @@ class TestModelGardenCredentialReuse: project_id="test-project", custom_llm_provider="vertex_ai", ) + + +@respx.mock +def test_sync_mistral_partner_completion_posts_to_raw_predict_without_a_client(): + route: Final = respx.post( + "https://us-central1-aiplatform.googleapis.com/v1/projects/partner-project/locations/us-central1" + "/publishers/mistralai/models/mistral-small-2503@001:rawPredict" + ).mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-vertex-mistral", + "object": "chat.completion", + "created": 1, + "model": "mistral-small-2503", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "I am Litellm Bot."}, + } + ], + "usage": {"prompt_tokens": 21, "completion_tokens": 5, "total_tokens": 26}, + }, + ) + ) + messages: Final = [ + {"role": "system", "content": "Your name is Litellm Bot, you are a helpful assistant"}, + {"role": "user", "content": "Hello, what is your name?"}, + ] + + with responses.RequestsMock() as google_token_endpoint: + token_route: Final = google_token_endpoint.post( + "https://oauth2.googleapis.com/token", + json={"access_token": "partner-access-token", "expires_in": 3600, "token_type": "Bearer"}, + ) + response: Final = litellm.completion( + model="vertex_ai/mistral-small-2503@001", + messages=messages, + vertex_project="partner-project", + vertex_location="us-central1", + vertex_credentials=json.dumps( + { + "type": "authorized_user", + "client_id": "partner-client-id", + "client_secret": "partner-client-secret", + "refresh_token": "partner-refresh-token", + } + ), + ) + + assert token_route.call_count == 1 + request: Final = route.calls[0].request + assert request.headers["authorization"] == "Bearer partner-access-token" + assert json.loads(request.content) == {"model": "mistral-small-2503", "messages": messages, "stream": False} + assert response.choices[0].message.content == "I am Litellm Bot." + assert response.usage.total_tokens == 26 diff --git a/tests/unit/llms/watsonx/embed/test_watsonx_embedding_transformation.py b/tests/unit/llms/watsonx/embed/test_watsonx_embedding_transformation.py index afcebcf0990..9fe5589b8a6 100644 --- a/tests/unit/llms/watsonx/embed/test_watsonx_embedding_transformation.py +++ b/tests/unit/llms/watsonx/embed/test_watsonx_embedding_transformation.py @@ -1,11 +1,16 @@ +import json +from typing import Final from unittest.mock import MagicMock import httpx import pytest +import respx from pydantic import ValidationError +import litellm + from litellm.llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig from litellm.types.utils import EmbeddingResponse @@ -132,3 +137,31 @@ def test_transform_embedding_response_shape_errors_do_not_echo_the_payload(paylo _transform(payload) assert "leaked payload text" not in str(exc_info.value) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_watsonx_embedding_sends_the_bearer_token_to_the_regional_endpoint( + sync_mode: bool, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("WATSONX_API_KEY", "mock-api-key") + monkeypatch.setenv("WATSONX_TOKEN", "mock-watsonx-token") + monkeypatch.setenv("WATSONX_API_BASE", "https://us-south.ml.cloud.ibm.com") + monkeypatch.setenv("WATSONX_PROJECT_ID", "mock-project-id") + route: Final = respx_mock.post(url__startswith="https://us-south.ml.cloud.ibm.com/ml/v1/text/embeddings").respond( + json={ + "model_id": "ibm/slate-30m-english-rtrvr", + "created_at": "2024-01-01T00:00:00.00Z", + "results": [{"embedding": [0.0, 1.0]}], + "input_token_count": 8, + } + ) + request: Final = {"model": "watsonx/ibm/slate-30m-english-rtrvr", "input": ["good morning from litellm"]} + + response: Final = litellm.embedding(**request) if sync_mode else await litellm.aembedding(**request) + + assert route.call_count == 1 + assert route.calls.last.request.headers["Authorization"] == "Bearer mock-watsonx-token" + assert json.loads(route.calls.last.request.content)["inputs"] == ["good morning from litellm"] + assert response.data == [{"object": "embedding", "index": 0, "embedding": [0.0, 1.0]}] + assert (response.usage.prompt_tokens, response.usage.total_tokens) == (8, 8) diff --git a/tests/unit/llms/watsonx/test_watsonx.py b/tests/unit/llms/watsonx/test_watsonx.py index c681a4bb1ce..4afa717debe 100644 --- a/tests/unit/llms/watsonx/test_watsonx.py +++ b/tests/unit/llms/watsonx/test_watsonx.py @@ -9,7 +9,7 @@ import pytest import litellm from litellm import completion, embedding -from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -370,6 +370,105 @@ def test_watsonx_tool_choice(sync_mode: bool, monkeypatch: pytest.MonkeyPatch) - assert len(response.choices) == 1 +async def test_watsonx_async_streamed_tool_choice_reaches_the_stream_endpoint(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("WATSONX_TOKEN", "mock-watsonx-token") + monkeypatch.setenv("WATSONX_API_BASE", "https://us-south.ml.cloud.ibm.com") + monkeypatch.setenv("WATSONX_PROJECT_ID", "mock-project-id") + stream_chunks: Final = ( + { + "id": "chatcmpl-watsonx", + "object": "chat.completion.chunk", + "created": 1760000000, + "model": "ibm/granite-3-8b-instruct", + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "id": "chatcmpl-tool-weather", + "type": "function", + "function": {"name": "get_current_weather", "arguments": ""}, + } + ], + }, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-watsonx", + "object": "chat.completion.chunk", + "created": 1760000000, + "model": "ibm/granite-3-8b-instruct", + "choices": [ + { + "index": 0, + "delta": { + "tool_calls": [{"index": 0, "function": {"arguments": '{"location": "San Francisco, CA"}'}}] + }, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-watsonx", + "object": "chat.completion.chunk", + "created": 1760000000, + "model": "ibm/granite-3-8b-instruct", + "choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}], + }, + ) + body: Final = "".join( + f"id: {index}\nevent: message\ndata: {json.dumps(chunk)}\n\n" for index, chunk in enumerate(stream_chunks) + ) + requests: Final[list[httpx.Request]] = [] + + def handle_request(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, text=body, headers={"content-type": "text/event-stream"}) + + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(handle_request)) + response: Final = await litellm.acompletion( + model="watsonx/ibm/granite-3-8b-instruct", + messages=[{"role": "user", "content": "What is the weather in San Francisco?"}], + tools=[ + { + "type": "function", + "function": { + "name": "get_current_weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, + } + ], + tool_choice="auto", + stream=True, + client=client, + ) + chunks: Final = [chunk async for chunk in response] + tool_calls: Final = tuple( + chunk.choices[0].delta.tool_calls[0] for chunk in chunks if chunk.choices[0].delta.tool_calls + ) + request_body: Final = cast(dict[str, object], json.loads(requests[0].content)) + + assert len(requests) == 1 + assert requests[0].url.path == "/ml/v1/text/chat_stream" + assert request_body["tool_choice_option"] == "auto" + assert request_body["stream"] is True + assert tool_calls[0].id == "chatcmpl-tool-weather" + assert tool_calls[0].function.name == "get_current_weather" + assert json.loads("".join(tool_call.function.arguments or "" for tool_call in tool_calls)) == { + "location": "San Francisco, CA" + } + assert chunks[-1].choices[0].finish_reason == "tool_calls" + + @pytest.mark.usefixtures("watsonx_env_vars", "_vcr_outcome_gate", "setup_and_teardown") def test_watsonx_chat_completions_endpoint_space_id(monkeypatch, watsonx_chat_completion_call): my_fake_space_id = "xxx-xxx-xxx-xxx-xxx" diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 7d01b1b2906..8dbef772472 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -1,3 +1,5 @@ +import json +import asyncio import datetime import time from collections.abc import Mapping @@ -6,12 +8,15 @@ from typing import Final, Literal, cast import httpx import pytest +import respx from openai.types.completion_usage import CompletionUsage from pydantic import BaseModel import litellm from litellm import TranscriptionResponse, model_cost from litellm.cost_calculator import BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, batch_cost_calculator, completion_cost, cost_per_token, handle_realtime_stream_cost_calculation, response_cost_calculator +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( convert_to_model_response_object, @@ -27,9 +32,41 @@ from litellm.types.utils import CallTypes, Choices, CompletionTokensDetailsWrapp from litellm.types.videos.main import VideoObject from litellm.utils import supports_prompt_caching import os +import threading from unittest.mock import MagicMock, patch +class ResponseCostCapture(CustomLogger): + def __init__(self) -> None: + self.success_cost: float | None = None + self.success_event: Final = threading.Event() + self.failure_cost: float | None = None + + def log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + response_cost: Final = kwargs.get("response_cost") + self.success_cost = float(response_cost) if isinstance(response_cost, (int, float)) else None + self.success_event.set() + + def log_failure_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.failure_cost = float(kwargs["response_cost"]) + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + response_cost: Final = kwargs.get("response_cost") + self.success_cost = float(response_cost) if isinstance(response_cost, (int, float)) else None + self.success_event.set() + + async def async_log_failure_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.failure_cost = float(kwargs["response_cost"]) + + @pytest.fixture def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") @@ -7923,6 +7960,380 @@ def test_cost_calculator_with_custom_pricing(): assert resp._hidden_params["response_cost"] > 0 +@pytest.fixture +def isolated_pricing_registry(local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "open_ai_chat_completion_models", set(litellm.open_ai_chat_completion_models)) + monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider)) + + +_CUSTOM_PRICING_MODEL: Final = "migration-custom-pricing-chat" +_CUSTOM_PRICING_RATES: Final = MappingProxyType( + {"input_cost_per_token": 0.0000008, "output_cost_per_token": 0.0000032} +) +_CUSTOM_PRICING_EXPECTED_COST: Final = 7 * 0.0000008 + 4 * 0.0000032 + + +def _custom_pricing_chunk(choices: list[dict[str, object]], usage: dict[str, int] | None = None) -> str: + chunk: Final = { + "id": "chatcmpl-custom-pricing", + "object": "chat.completion.chunk", + "created": 1, + "model": _CUSTOM_PRICING_MODEL, + "choices": choices, + **({"usage": usage} if usage is not None else {}), + } + return f"data: {json.dumps(chunk)}\n\n" + + +def _custom_pricing_upstream(request: httpx.Request) -> httpx.Response: + if json.loads(request.content).get("stream") is True: + body: Final = "".join( + ( + _custom_pricing_chunk( + [{"index": 0, "delta": {"role": "assistant", "content": "I am ready to help."}, "finish_reason": None}] + ), + _custom_pricing_chunk([{"index": 0, "delta": {}, "finish_reason": "stop"}]), + _custom_pricing_chunk([], {"prompt_tokens": 7, "completion_tokens": 4, "total_tokens": 11}), + "data: [DONE]\n\n", + ) + ) + return httpx.Response(200, text=body, headers={"content-type": "text/event-stream"}) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-custom-pricing", + "object": "chat.completion", + "created": 1, + "model": _CUSTOM_PRICING_MODEL, + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "I am ready to help."}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 7, "completion_tokens": 4, "total_tokens": 11}, + }, + ) + + +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.asyncio +@pytest.mark.usefixtures("isolated_pricing_registry") +@respx.mock +async def test_custom_pricing_matches_response_usage(async_mode: bool) -> None: + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock(side_effect=_custom_pricing_upstream) + request: Final = { + "model": f"openai/{_CUSTOM_PRICING_MODEL}", + "messages": [{"role": "user", "content": "How are you?"}], + "api_key": "sk-test", + **_CUSTOM_PRICING_RATES, + } + + response: Final = await litellm.acompletion(**request) if async_mode else litellm.completion(**request) + + assert len(route.calls) == 1 + assert response._hidden_params["response_cost"] == pytest.approx(_CUSTOM_PRICING_EXPECTED_COST) + + +@pytest.mark.usefixtures("isolated_pricing_registry") +@respx.mock +def test_streamed_custom_pricing_cost_matches_non_streamed_cost() -> None: + respx.post("https://api.openai.com/v1/chat/completions").mock(side_effect=_custom_pricing_upstream) + request: Final = { + "model": f"openai/{_CUSTOM_PRICING_MODEL}", + "messages": [{"role": "user", "content": "Hi"}], + "api_key": "sk-test", + **_CUSTOM_PRICING_RATES, + } + + non_streamed: Final = litellm.completion(**request) + rebuilt: Final = litellm.stream_chunk_builder( + list(litellm.completion(**request, stream=True, stream_options={"include_usage": True})), + messages=request["messages"], + ) + streamed_cost: Final = completion_cost(rebuilt, custom_cost_per_token=_CUSTOM_PRICING_RATES) + + assert rebuilt.choices[0].message.content == non_streamed.choices[0].message.content == "I am ready to help." + assert non_streamed._hidden_params["response_cost"] == pytest.approx(_CUSTOM_PRICING_EXPECTED_COST) + assert streamed_cost == pytest.approx(non_streamed._hidden_params["response_cost"]) + + +@pytest.mark.parametrize( + ("model", "provider"), + [ + ("fireworks_ai/accounts/fireworks/models/custom-migration-model", "fireworks_ai"), + ("vertex_ai/meta/llama-custom-migration-model", "vertex_ai"), + ("azure/custom-migration-model", "azure"), + ("mistral/mistral-large-latest", "mistral"), + ], +) +def test_custom_cost_uses_test_owned_rates_across_providers(model: str, provider: str): + response: Final = litellm.ModelResponse( + model=model, + choices=[], + usage=litellm.Usage(prompt_tokens=9, completion_tokens=4, total_tokens=13), + ) + rates: Final = MappingProxyType({"input_cost_per_token": 0.002, "output_cost_per_token": 0.004}) + + cost: Final = completion_cost( + completion_response=response, + model=model, + custom_llm_provider=provider, + custom_cost_per_token=rates, + ) + + assert cost == pytest.approx(0.034) + + +@respx.mock +def test_fireworks_completion_cost_uses_registered_rate(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + model: Final = "fireworks_ai/accounts/fireworks/models/migration-model" + monkeypatch.setitem( + litellm.model_cost, + model, + {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002, "litellm_provider": "fireworks_ai", "mode": "chat"}, + ) + respx.post("https://api.fireworks.ai/inference/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-fireworks", + "object": "chat.completion", + "created": 1, + "model": "accounts/fireworks/models/migration-model", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Going great"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 8, "completion_tokens": 5, "total_tokens": 13}, + }, + ) + ) + + response: Final = litellm.completion( + model=model, messages=[{"role": "user", "content": "Hey, how's it going?"}], api_key="fw-test" + ) + + assert completion_cost(completion_response=response) == pytest.approx(0.018) + + +def test_azure_embedding_cost_uses_registered_base_model_rate(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem( + litellm.model_cost, + "azure/migration-embedding-model", + { + "input_cost_per_token": 0.002, + "output_cost_per_token": 0.0, + "litellm_provider": "azure", + "mode": "embedding", + }, + ) + + cost: Final = response_cost_calculator( + response_object=EmbeddingResponse( + model="text-embedding-3-small", + data=[{"embedding": [1, 2, 3]}], + usage=Usage(prompt_tokens=10, completion_tokens=10), + ), + model="text-embedding-3-small", + custom_llm_provider=None, + base_model="azure/migration-embedding-model", + call_type="aembedding", + optional_params={}, + custom_pricing=False, + prompt="Hello, world!", + ) + + assert cost == pytest.approx(0.02) + + +def test_azure_common_deployment_uses_registered_base_model_pricing(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem( + litellm.model_cost, + "azure/migration-pricing-model", + { + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + "litellm_provider": "azure", + "mode": "chat", + }, + ) + response: Final = ModelResponse( + id="chatcmpl-migration", + choices=[Choices(finish_reason="stop", index=0, message=Message(content="done", role="assistant"))], + created=1, + model="migration-deployment", + object="chat.completion", + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) + + cost: Final = response_cost_calculator( + response_object=response, + model="migration-deployment", + custom_llm_provider="azure", + call_type="acompletion", + optional_params={}, + base_model="azure/migration-pricing-model", + ) + + assert cost == pytest.approx(0.02) + + +def test_azure_tts_cost_uses_test_owned_character_rate(monkeypatch: pytest.MonkeyPatch) -> None: + model: Final = "migration-tts-model" + monkeypatch.setitem( + litellm.model_cost, + f"azure/{model}", + { + "input_cost_per_character": 0.01, + "output_cost_per_character": 0.0, + "litellm_provider": "azure", + "mode": "audio_speech", + }, + ) + + cost: Final = response_cost_calculator( + response_object=TranscriptionResponse(text=""), + model=model, + custom_llm_provider="azure", + call_type="aspeech", + optional_params={}, + prompt="hello", + ) + + assert cost == pytest.approx(0.05) + + +def test_moderation_cost_uses_test_owned_token_rate() -> None: + prompt: Final = "moderation input" + rates: Final = MappingProxyType( + {"input_cost_per_token": 0.001, "output_cost_per_token": 0.0} + ) + + cost: Final = completion_cost( + completion_response=None, + model="omni-moderation-latest", + custom_llm_provider="openai", + call_type="moderation", + prompt=prompt, + custom_cost_per_token=rates, + ) + expected_cost: Final = litellm.token_counter(model="omni-moderation-latest", text=prompt) * 0.001 + + assert cost == pytest.approx(expected_cost) + + +def test_vertex_predictive_cost_uses_test_owned_token_rate() -> None: + model: Final = "vertex_ai/migration-llama" + messages: Final = [{"role": "user", "content": "price control"}] + rates: Final = MappingProxyType( + {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002} + ) + + cost: Final = completion_cost( + model=model, + messages=messages, + custom_llm_provider="vertex_ai", + custom_cost_per_token=rates, + ) + expected_cost: Final = ( + litellm.token_counter(model=model, messages=messages) * 0.001 + ) + + assert cost == pytest.approx(expected_cost) + + +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.asyncio +@pytest.mark.usefixtures("isolated_pricing_registry") +@respx.mock +async def test_custom_pricing_cost_reaches_success_callback( + async_mode: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + model: Final = "migration-callback-pricing-model" + litellm.register_model( + { + model: { + "input_cost_per_token": 0.0001, + "output_cost_per_token": 0.0002, + "litellm_provider": "openai", + "mode": "chat", + } + }, + persist_across_reloads=False, + ) + callback: Final = ResponseCostCapture() + if async_mode: + monkeypatch.setattr(litellm, "_async_success_callback", [callback]) + else: + monkeypatch.setattr(litellm, "success_callback", [callback]) + response_data: Final = { + "id": "chatcmpl-priced", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "response"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 8, "completion_tokens": 5, "total_tokens": 13}, + } + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, json=response_data) + ) + request: Final = { + "model": model, + "messages": [{"role": "user", "content": "price control"}], + "api_key": "test-openai-key", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + } + response: Final = ( + await litellm.acompletion(**request) if async_mode else litellm.completion(**request) + ) + await GLOBAL_LOGGING_WORKER.flush() + + assert await asyncio.to_thread(callback.success_event.wait, 10.0) + assert len(route.calls) == 1 + assert response.choices[0].message.content == "response" + assert callback.success_cost == pytest.approx(0.018) + + +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.asyncio +async def test_failure_completion_cost_reaches_failure_callback( + async_mode: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + callback: Final = ResponseCostCapture() + monkeypatch.setattr(litellm, "success_callback", [callback]) + monkeypatch.setattr(litellm, "_async_success_callback", [callback]) + monkeypatch.setattr(litellm, "failure_callback", [callback]) + monkeypatch.setattr(litellm, "_async_failure_callback", [callback]) + failure: Final = litellm.InternalServerError( + message="scripted failure", + llm_provider="openai", + model="gpt-6-sol", + ) + + if async_mode: + with pytest.raises(litellm.InternalServerError): + await litellm.acompletion( + model="gpt-6-sol", + messages=[{"role": "user", "content": "fail"}], + mock_response=failure, + ) + else: + with pytest.raises(litellm.InternalServerError): + litellm.completion( + model="gpt-6-sol", + messages=[{"role": "user", "content": "fail"}], + mock_response=failure, + ) + + assert callback.failure_cost == 0.0 + + @pytest.fixture def model_item(): return { diff --git a/tests/unit/test_docker_no_network_on_deploy.py b/tests/unit/test_docker_no_network_on_deploy.py new file mode 100644 index 00000000000..0bccdca7baa --- /dev/null +++ b/tests/unit/test_docker_no_network_on_deploy.py @@ -0,0 +1,17 @@ +from pathlib import Path +from typing import Final + +_NETWORK_FETCH_TOKENS: Final = ("curl", "wget", "fetch", "http://", "https://") + + +def test_container_build_no_network_fetch() -> None: + dockerfile: Final = (Path(__file__).resolve().parents[2] / "Dockerfile").read_text() + startup_lines: Final = tuple( + line.strip() for line in dockerfile.splitlines() if line.strip().upper().startswith(("CMD", "ENTRYPOINT")) + ) + fetching_lines: Final = tuple( + line for line in startup_lines if any(token in line.lower() for token in _NETWORK_FETCH_TOKENS) + ) + + assert startup_lines != () + assert fetching_lines == () diff --git a/tests/unit/test_fake_openai_endpoint.py b/tests/unit/test_fake_openai_endpoint.py new file mode 100644 index 00000000000..de9c560624a --- /dev/null +++ b/tests/unit/test_fake_openai_endpoint.py @@ -0,0 +1,51 @@ +import re +from pathlib import Path +from typing import Final + +import pytest + +from tests.fake_openai_endpoint import _LOCAL_DEFAULT, _resolve_base + +_TESTS_ROOT: Final = Path(__file__).resolve().parents[1] +_LIVE_HOSTED_MOCK: Final = re.compile(r"railway\.app(?:/|\")") +_LOCAL_FAKE_IMPORT: Final = "fake_openai_endpoint" + + +def _local_fake_users_with_hosted_mock(root: Path) -> tuple[Path, ...]: + sources: Final = ((path, path.read_text()) for path in sorted(root.rglob("*.py"))) + return tuple( + path for path, source in sources if _LOCAL_FAKE_IMPORT in source and _LIVE_HOSTED_MOCK.search(source) + ) + + +@pytest.mark.parametrize("host", ("127.0.0.1", "localhost", "[::1]")) +def test_loopback_env_base_is_honored(monkeypatch: pytest.MonkeyPatch, host: str) -> None: + base: Final = f"http://{host}:9191" + monkeypatch.setenv("FAKE_OPENAI_API_BASE", base) + + assert _resolve_base() == base + + +def test_remote_env_base_resolves_to_local(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FAKE_OPENAI_API_BASE", "https://remote.example.invalid") + + assert _resolve_base() == _LOCAL_DEFAULT + + +def test_migrated_files_have_no_live_hosted_mock() -> None: + offenders: Final = _local_fake_users_with_hosted_mock(_TESTS_ROOT) + + assert offenders == (), [str(path.relative_to(_TESTS_ROOT)) for path in offenders] + + +def test_hosted_mock_scan_flags_only_local_fake_users(tmp_path: Path) -> None: + host: Final = ".".join(("railway", "app")) + both: Final = tmp_path / "integration/uses_fake_and_hosted.py" + both.parent.mkdir(parents=True) + both.write_text(f'from tests.{_LOCAL_FAKE_IMPORT} import FAKE_OPENAI_API_BASE\napi_base = "https://mock.{host}"\n') + (tmp_path / "integration/uses_fake_only.py").write_text( + f"from tests.{_LOCAL_FAKE_IMPORT} import FAKE_OPENAI_API_BASE\n" + ) + (tmp_path / "integration/hosted_string_only.py").write_text(f'api_base = "https://mock.{host}/"\n') + + assert _local_fake_users_with_hosted_mock(tmp_path) == (both,) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 5f0c40b3eab..8af10b55648 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -6,8 +6,10 @@ import io import json import logging import os +import threading import urllib.parse from collections.abc import Mapping +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime from importlib import import_module @@ -2015,6 +2017,186 @@ def test_stream_chunk_builder_keeps_tool_calls_carried_only_by_a_later_choice_of ] +def test_parallel_function_call_stream_reassembles_each_arguments_payload(): + from litellm import stream_chunk_builder + from litellm.types.utils import ChatCompletionDeltaToolCall, Delta, Function, ModelResponseStream, StreamingChoices + + def chunk(tool_calls: list[ChatCompletionDeltaToolCall], finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-parallel-tools", + created=1751934860, + model="gpt-4.1-mini", + object="chat.completion.chunk", + choices=[ + StreamingChoices(index=0, delta=Delta(tool_calls=tool_calls), finish_reason=finish_reason) + ], + ) + + chunks: Final = [ + chunk( + [ + ChatCompletionDeltaToolCall( + id="call_weather", + index=0, + type="function", + function=Function(name="weather", arguments='{"city":"'), + ), + ChatCompletionDeltaToolCall( + id="call_time", + index=1, + type="function", + function=Function(name="time", arguments='{"zone":"'), + ), + ] + ), + chunk( + [ + ChatCompletionDeltaToolCall(index=0, function=Function(arguments='Paris"}')), + ChatCompletionDeltaToolCall(index=1, function=Function(arguments='UTC"}')), + ], + finish_reason="tool_calls", + ), + ] + + response: Final = stream_chunk_builder(chunks=chunks) + tool_calls: Final = response.choices[0].message.tool_calls + + assert tool_calls is not None + assert [(call.id, call.function.name, call.function.arguments) for call in tool_calls] == [ + ("call_weather", "weather", '{"city":"Paris"}'), + ("call_time", "time", '{"zone":"UTC"}'), + ] + + +def _responses_sse(events: tuple[Mapping[str, object], ...]) -> bytes: + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode() + + +@respx.mock +def test_parallel_function_call_stream_over_responses_bridge_ignores_events_it_does_not_translate(): + response_stub: Final = { + "id": "resp_parallel", + "object": "response", + "created_at": 1, + "status": "in_progress", + "model": "gpt-6-luna", + "output": [], + } + sf_args: Final = '{"location":"San Francisco, CA"}' + tokyo_args: Final = '{"location":"Tokyo"}' + sf_call: Final = { + "type": "function_call", + "id": "fc_sf", + "call_id": "call_sf", + "name": "get_current_weather", + "arguments": "", + "status": "in_progress", + } + tokyo_call: Final = {**sf_call, "id": "fc_tokyo", "call_id": "call_tokyo"} + events: Final = ( + {"type": "response.created", "sequence_number": 0, "response": response_stub}, + {"type": "response.in_progress", "sequence_number": 1, "response": response_stub}, + {"type": "response.output_item.added", "sequence_number": 2, "output_index": 0, "item": sf_call}, + { + "type": "response.function_call_arguments.delta", + "sequence_number": 3, + "item_id": "fc_sf", + "output_index": 0, + "delta": sf_args[:12], + }, + { + "type": "response.function_call_arguments.delta", + "sequence_number": 4, + "item_id": "fc_sf", + "output_index": 0, + "delta": sf_args[12:], + }, + { + "type": "response.function_call_arguments.done", + "sequence_number": 5, + "item_id": "fc_sf", + "output_index": 0, + "arguments": sf_args, + }, + { + "type": "response.output_item.done", + "sequence_number": 6, + "output_index": 0, + "item": {**sf_call, "arguments": sf_args, "status": "completed"}, + }, + {"type": "response.output_item.added", "sequence_number": 7, "output_index": 1, "item": tokyo_call}, + { + "type": "response.function_call_arguments.delta", + "sequence_number": 8, + "item_id": "fc_tokyo", + "output_index": 1, + "delta": tokyo_args, + }, + { + "type": "response.function_call_arguments.done", + "sequence_number": 9, + "item_id": "fc_tokyo", + "output_index": 1, + "arguments": tokyo_args, + }, + { + "type": "response.output_item.done", + "sequence_number": 10, + "output_index": 1, + "item": {**tokyo_call, "arguments": tokyo_args, "status": "completed"}, + }, + { + "type": "response.completed", + "sequence_number": 11, + "response": { + **response_stub, + "status": "completed", + "output": [ + {**sf_call, "arguments": sf_args, "status": "completed"}, + {**tokyo_call, "arguments": tokyo_args, "status": "completed"}, + ], + "usage": {"input_tokens": 80, "output_tokens": 40, "total_tokens": 120}, + }, + }, + ) + route: Final = respx.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response( + 200, headers={"content-type": "text/event-stream"}, content=_responses_sse(events) + ) + ) + + response: Final = litellm.completion( + model="gpt-6-luna", + api_key="sk-test", + api_base="https://api.openai.com/v1", + messages=[{"role": "user", "content": "What's the weather like in San Francisco and Tokyo?"}], + tools=[ + { + "type": "function", + "function": { + "name": "get_current_weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, + } + ], + stream=True, + complete_response=True, + ) + + assert route.call_count == 1 + message: Final = response.choices[0].message + assert message.content is None + assert [(call.id, call.function.name, call.function.arguments) for call in message.tool_calls] == [ + ("call_sf", "get_current_weather", sf_args), + ("call_tokyo", "get_current_weather", tokyo_args), + ] + assert response.choices[0].finish_reason == "tool_calls" + + def test_stream_chunk_builder_thinking_blocks(): from litellm import stream_chunk_builder from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices @@ -2593,132 +2775,108 @@ async def test_retrying() -> None: ) -def test_anthropic_disable_url_suffix_env_var(): - """Test that LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX prevents /v1/messages suffix.""" - import os - from unittest.mock import MagicMock, patch +def test_anthropic_disable_url_suffix_env_var( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setenv("ANTHROPIC_API_BASE", "https://api.example.com") + default_route: Final = respx_mock.post("https://api.example.com/v1/messages").mock( + return_value=httpx.Response( + 200, + json={ + "id": "msg_test", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "test response"}], + "model": "claude-3-sonnet", + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 2}, + }, + ) + ) + default_response: Final = litellm.completion( + model="anthropic/claude-3-sonnet", + messages=[{"role": "user", "content": "test"}], + api_key="test-key", + ) - from litellm import completion + assert default_response.choices[0].message.content == "test response" + assert default_route.called + assert str(default_route.calls.last.request.url) == "https://api.example.com/v1/messages" - # Test with environment variable disabled (default behavior) - with patch.dict(os.environ, {"ANTHROPIC_API_BASE": "https://api.example.com"}): - actual_api_base = None + monkeypatch.setenv("ANTHROPIC_API_BASE", "https://api.example.com/custom/path") + monkeypatch.setenv("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX", "true") + disabled_route: Final = respx_mock.post("https://api.example.com/custom/path").mock( + return_value=httpx.Response( + 200, + json={ + "id": "msg_test", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "test response"}], + "model": "claude-3-sonnet", + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 2}, + }, + ) + ) + disabled_response: Final = litellm.completion( + model="anthropic/claude-3-sonnet", + messages=[{"role": "user", "content": "test"}], + api_key="test-key", + ) - with patch("litellm.main.anthropic_chat_completions") as mock_anthropic: - - def capture_completion(**kwargs): - nonlocal actual_api_base - actual_api_base = kwargs.get("api_base") - mock_response = MagicMock() - mock_response.choices = [MagicMock()] - return mock_response - - mock_anthropic.completion = capture_completion - - # This should append /v1/messages - completion( - model="anthropic/claude-3-sonnet", - messages=[{"role": "user", "content": "test"}], - api_key="test-key", - ) - - # Verify the api_base has /v1/messages appended - assert actual_api_base.endswith("/v1/messages") - assert actual_api_base == "https://api.example.com/v1/messages" - - # Test with environment variable enabled - with patch.dict( - os.environ, - { - "ANTHROPIC_API_BASE": "https://api.example.com/custom/path", - "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX": "true", - }, - ): - actual_api_base = None - - with patch("litellm.main.anthropic_chat_completions") as mock_anthropic: - - def capture_completion(**kwargs): - nonlocal actual_api_base - actual_api_base = kwargs.get("api_base") - mock_response = MagicMock() - mock_response.choices = [MagicMock()] - return mock_response - - mock_anthropic.completion = capture_completion - - # This should NOT append /v1/messages - completion( - model="anthropic/claude-3-sonnet", - messages=[{"role": "user", "content": "test"}], - api_key="test-key", - ) - - # Verify the api_base does not have /v1/messages appended - assert actual_api_base == "https://api.example.com/custom/path" - assert not actual_api_base.endswith("/v1/messages") + assert disabled_response.choices[0].message.content == "test response" + assert disabled_route.called + assert str(disabled_route.calls.last.request.url) == "https://api.example.com/custom/path" -def test_anthropic_text_disable_url_suffix_env_var(): - """Test that LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX prevents /v1/complete suffix for anthropic_text.""" - import os - from unittest.mock import MagicMock, patch +def test_anthropic_text_disable_url_suffix_env_var( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setenv("ANTHROPIC_API_BASE", "https://api.example.com") + default_route: Final = respx_mock.post("https://api.example.com/v1/complete").mock( + return_value=httpx.Response( + 200, + json={ + "completion": "test response", + "stop_reason": "stop_sequence", + "model": "claude-instant-1", + }, + ) + ) + default_response: Final = litellm.text_completion( + model="anthropic_text/claude-instant-1", + prompt="test", + api_key="test-key", + ) - from litellm import completion + assert default_response.choices[0].text == "test response" + assert default_route.called + assert str(default_route.calls.last.request.url) == "https://api.example.com/v1/complete" - # Test with environment variable disabled (default behavior) - with patch.dict(os.environ, {"ANTHROPIC_API_BASE": "https://api.example.com"}): - actual_api_base = None + monkeypatch.setenv("ANTHROPIC_API_BASE", "https://api.example.com/custom/complete") + monkeypatch.setenv("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX", "true") + disabled_route: Final = respx_mock.post("https://api.example.com/custom/complete").mock( + return_value=httpx.Response( + 200, + json={ + "completion": "test response", + "stop_reason": "stop_sequence", + "model": "claude-instant-1", + }, + ) + ) + disabled_response: Final = litellm.text_completion( + model="anthropic_text/claude-instant-1", + prompt="test", + api_key="test-key", + ) - with patch("litellm.main.base_llm_http_handler") as mock_handler: - - def capture_completion(**kwargs): - nonlocal actual_api_base - actual_api_base = kwargs.get("api_base") - return MagicMock() - - mock_handler.completion = capture_completion - - # This should append /v1/complete - completion( - model="anthropic_text/claude-instant-1", - messages=[{"role": "user", "content": "test"}], - api_key="test-key", - ) - - # Verify the api_base has /v1/complete appended - assert actual_api_base.endswith("/v1/complete") - assert actual_api_base == "https://api.example.com/v1/complete" - - # Test with environment variable enabled - with patch.dict( - os.environ, - { - "ANTHROPIC_API_BASE": "https://api.example.com/custom/complete", - "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX": "true", - }, - ): - actual_api_base = None - - with patch("litellm.main.base_llm_http_handler") as mock_handler: - - def capture_completion(**kwargs): - nonlocal actual_api_base - actual_api_base = kwargs.get("api_base") - return MagicMock() - - mock_handler.completion = capture_completion - - # This should NOT append /v1/complete - completion( - model="anthropic_text/claude-instant-1", - messages=[{"role": "user", "content": "test"}], - api_key="test-key", - ) - - # Verify the api_base does not have /v1/complete appended - assert actual_api_base == "https://api.example.com/custom/complete" - assert not actual_api_base.endswith("/v1/complete") + assert disabled_response.choices[0].text == "test response" + assert disabled_route.called + assert str(disabled_route.calls.last.request.url) == "https://api.example.com/custom/complete" def test_image_edit_merges_headers_and_extra_headers(): @@ -5293,26 +5451,1255 @@ def test_completion_anthropic_hanging(): messages, model="claude-3-sonnet-20240229", llm_provider="anthropic" ) - print(f"converted_messages: {converted_messages}") - - ## ENSURE USER / ASSISTANT ALTERNATING + assert len(converted_messages) == 3 for i, msg in enumerate(converted_messages): if i < len(converted_messages) - 1: assert msg["role"] != converted_messages[i + 1]["role"] +@respx.mock +def test_completion_openai_returns_the_scripted_assistant_message(): + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-migration-openai", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "scripted answer"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, + }, + ) + ) + + response: Final = litellm.completion( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + api_key="test-openai-key", + ) + + assert response.choices[0].message.content == "scripted answer" + assert route.calls.last.request.headers["Authorization"] == "Bearer test-openai-key" + + +@respx.mock +def test_completion_openai_response_headers(monkeypatch: pytest.MonkeyPatch, openai_api_response): + def respond(request: httpx.Request) -> httpx.Response: + request_body: Final = json.loads(request.content) + headers: Final = { + "x-ratelimit-remaining-tokens": "99", + "x-ratelimit-remaining-requests": "9", + } + if request_body.get("stream"): + chunks: Final = ( + { + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "created": 1, + "model": request_body["model"], + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "Hello!"}, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "created": 1, + "model": request_body["model"], + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + ) + body: Final = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n" + return httpx.Response( + 200, + text=body, + headers={**headers, "content-type": "text/event-stream"}, + ) + return httpx.Response(200, json=openai_api_response, headers=headers) + + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock(side_effect=respond) + monkeypatch.setattr(litellm, "return_response_headers", True) + + response: Final = litellm.completion( + model="gpt-6-sol", + messages=[{"role": "user", "content": "hello"}], + ) + + assert len(route.calls) == 1 + assert response._response_headers["x-ratelimit-remaining-tokens"] == "99" + assert response._hidden_params["additional_headers"]["x-ratelimit-remaining-requests"] == "9" + assert response._hidden_params["additional_headers"]["llm_provider-x-ratelimit-remaining-requests"] == "9" + stream: Final = litellm.completion( + model="gpt-6-sol", + messages=[{"role": "user", "content": "hello"}], + stream=True, + ) + chunks: Final = list(stream) + assert len(route.calls) == 2 + assert stream._response_headers["x-ratelimit-remaining-tokens"] == "99" + assert stream._hidden_params["additional_headers"]["x-ratelimit-remaining-requests"] == "9" + assert chunks[0].choices[0].delta.content == "Hello!" + + respx.post("https://api.openai.com/v1/embeddings").mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + }, + headers={"x-ratelimit-remaining-tokens": "98"}, + ) + ) + embedding: Final = litellm.embedding(model="text-embedding-3-small", input="hello") + assert embedding._response_headers["x-ratelimit-remaining-tokens"] == "98" + + +@pytest.mark.asyncio +@respx.mock +async def test_async_completion_openai_response_headers(monkeypatch: pytest.MonkeyPatch, openai_api_response): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + stream_body: Final = ( + "data: " + + json.dumps( + { + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-6-sol", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}], + } + ) + + "\n\ndata: [DONE]\n\n" + ) + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + side_effect=( + httpx.Response(200, json=openai_api_response, headers={"x-ratelimit-remaining-tokens": "99"}), + httpx.Response( + 200, + text=stream_body, + headers={"x-ratelimit-remaining-tokens": "97", "content-type": "text/event-stream"}, + ), + ) + ) + respx.post("https://api.openai.com/v1/embeddings").mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + }, + headers={"x-ratelimit-remaining-tokens": "98"}, + ) + ) + monkeypatch.setattr(litellm, "return_response_headers", True) + + response: Final = await litellm.acompletion( + model="gpt-6-sol", + messages=[{"role": "user", "content": "hello"}], + ) + stream: Final = await litellm.acompletion( + model="gpt-6-sol", + messages=[{"role": "user", "content": "hello"}], + stream=True, + ) + stream_text: Final = "".join([chunk.choices[0].delta.content or "" async for chunk in stream]) + embedding: Final = await litellm.aembedding(model="text-embedding-3-small", input="hello") + + assert len(route.calls) == 2 + assert response._response_headers["x-ratelimit-remaining-tokens"] == "99" + assert stream._response_headers["x-ratelimit-remaining-tokens"] == "97" + assert stream_text == "Hi" + assert embedding._response_headers["x-ratelimit-remaining-tokens"] == "98" + + +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.asyncio +@respx.mock +async def test_azure_rate_limit_error_carries_retry_after_header(async_mode: bool, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + respx.post("https://example.openai.azure.com/openai/deployments/text-embedding-3-small/embeddings").mock( + return_value=httpx.Response( + 429, + json={"error": {"message": "Rate Limit Error!", "code": "429"}}, + headers={"retry-after": "30"}, + ) + ) + request: Final = { + "model": "azure/text-embedding-3-small", + "input": "hello", + "api_base": "https://example.openai.azure.com", + "api_version": "2024-02-01", + "api_key": "azure-test-key", + "max_retries": 0, + } + + with pytest.raises(litellm.RateLimitError) as error: + await (litellm.aembedding(**request) if async_mode else asyncio.to_thread(litellm.embedding, **request)) + + assert error.value.litellm_response_headers["retry-after"] == "30" + + +@respx.mock +def test_completion_preserves_empty_openai_message_content(): + response_data: Final = { + "id": "chatcmpl-empty", + "object": "chat.completion", + "created": 1, + "model": "gpt-6-sol", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": ""}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 0, "total_tokens": 1}, + } + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, json=response_data) + ) + + response: Final = litellm.completion( + model="gpt-6-sol", + messages=[{"role": "user", "content": ""}], + ) + + assert len(route.calls) == 1 + request_body: Final = json.loads(route.calls[0].request.content) + assert request_body == { + "model": "gpt-6-sol", + "messages": [{"role": "user", "content": ""}], + } + assert response.choices[0].message.content == "" + + +@respx.mock +def test_openai_completion_parses_scripted_chat_response(openai_api_response): + response_data: Final = {**openai_api_response, "model": "gpt-6-sol"} + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, json=response_data) + ) + messages: Final = [{"role": "user", "content": "hello"}] + + response: Final = litellm.completion(model="gpt-6-sol", messages=messages) + + request_body: Final = json.loads(route.calls[0].request.content) + assert request_body == {"model": "gpt-6-sol", "messages": messages} + assert response.choices[0].message.content == response_data["choices"][0]["message"]["content"] + assert response.usage.total_tokens == response_data["usage"]["total_tokens"] + + +@respx.mock +def test_completion_logprobs_parses_top_logprobs(): + response_data: Final = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "gpt-6-sol", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "hello"}, + "logprobs": { + "content": [ + { + "token": "hello", + "logprob": -0.1, + "bytes": [104, 101, 108, 108, 111], + "top_logprobs": [ + {"token": "hello", "logprob": -0.1, "bytes": [104, 101, 108, 108, 111]}, + {"token": "hi", "logprob": -0.2, "bytes": [104, 105]}, + {"token": "hey", "logprob": -0.3, "bytes": [104, 101, 121]}, + ], + } + ] + }, + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, json=response_data) + ) + + response: Final = litellm.completion( + model="gpt-6-sol", + messages=[{"role": "user", "content": "hello"}], + logprobs=True, + top_logprobs=3, + reasoning_effort="none", + ) + + request_body: Final = json.loads(route.calls[0].request.content) + assert request_body == { + "model": "gpt-6-sol", + "messages": [{"role": "user", "content": "hello"}], + "logprobs": True, + "top_logprobs": 3, + "reasoning_effort": "none", + } + assert [ + item.token for item in response.choices[0].logprobs.content[0].top_logprobs + ] == ["hello", "hi", "hey"] + choice: Final = response.choices[0] + assert "logprobs" in choice + assert "content" in choice.logprobs + assert "delta" not in choice + assert "token" not in choice.logprobs + + +@respx.mock +def test_completion_logprobs_stream_parses_top_logprobs(): + chunks: Final = ( + { + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-6-sol", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "hello"}, + "finish_reason": None, + "logprobs": { + "content": [ + { + "token": "hello", + "logprob": -0.1, + "bytes": [104, 101, 108, 108, 111], + "top_logprobs": [ + {"token": "hello", "logprob": -0.1, "bytes": [104, 101, 108, 108, 111]}, + {"token": "hi", "logprob": -0.2, "bytes": [104, 105]}, + {"token": "hey", "logprob": -0.3, "bytes": [104, 101, 121]}, + ], + } + ] + }, + } + ], + }, + { + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-6-sol", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop", "logprobs": {"content": []}}], + }, + ) + stream_body: Final = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n" + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, text=stream_body, headers={"content-type": "text/event-stream"}) + ) + + streamed_response: Final = litellm.completion( + model="gpt-6-sol", + messages=[{"role": "user", "content": "hello"}], + logprobs=True, + top_logprobs=3, + reasoning_effort="none", + stream=True, + ) + response_chunks: Final = list(streamed_response) + + assert len(route.calls) == 1 + stream_request_body: Final = json.loads(route.calls[0].request.content) + assert stream_request_body == { + "model": "gpt-6-sol", + "messages": [{"role": "user", "content": "hello"}], + "logprobs": True, + "top_logprobs": 3, + "reasoning_effort": "none", + "stream_options": {"include_usage": True}, + "stream": True, + } + assert [ + item.token for item in response_chunks[0].choices[0].logprobs.content[0].top_logprobs + ] == ["hello", "hi", "hey"] + + +@respx.mock +def test_completion_fireworks_ai_parses_response(): + response_data: Final = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "accounts/fireworks/models/deepseek-v4-pro-0813", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Hello there!"}, + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12}, + } + route: Final = respx.post("https://api.fireworks.ai/inference/v1/chat/completions").mock( + return_value=httpx.Response(200, json=response_data) + ) + messages: Final = [{"role": "user", "content": "hello"}] + + response: Final = litellm.completion( + model="fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813", + messages=messages, + api_key="fireworks-test-key", + ) + + request_body: Final = json.loads(route.calls[0].request.content) + assert request_body == { + "model": "accounts/fireworks/models/deepseek-v4-pro-0813", + "messages": messages, + } + assert response.choices[0].message.content == "Hello there!" + assert response.usage.total_tokens == 12 + + +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.asyncio +@respx.mock +async def test_bedrock_converse_parses_scripted_response( + async_mode: bool, monkeypatch: pytest.MonkeyPatch, fake_provider_credentials: None +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + route: Final = respx.post( + "https://bedrock-runtime.us-west-2.amazonaws.com/model/us.anthropic.claude-sonnet-5-5/converse" + ).mock( + return_value=httpx.Response( + 200, + json={ + "output": { + "message": { + "role": "assistant", + "content": [{"text": "Hello from Bedrock."}], + } + }, + "stopReason": "end_turn", + "usage": {"inputTokens": 8, "outputTokens": 5, "totalTokens": 13}, + "metrics": {"latencyMs": 12}, + }, + ) + ) + request: Final = { + "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", + "messages": [{"role": "user", "content": "hello"}], + "api_key": "test-bedrock-token", + "aws_region_name": "us-west-2", + } + response: Final = ( + await litellm.acompletion(**request) if async_mode else litellm.completion(**request) + ) + + assert route.calls[0].request.headers["authorization"] == "Bearer test-bedrock-token" + assert json.loads(route.calls[0].request.content) == { + "messages": [{"role": "user", "content": [{"text": "hello"}]}], + "inferenceConfig": {}, + } + assert response.choices[0].message.content == "Hello from Bedrock." + assert response.usage.prompt_tokens == 8 + assert response.usage.completion_tokens == 5 + + +@respx.mock +def test_bedrock_converse_parses_scripted_tool_call_arguments( + fake_provider_credentials: None, +): + route: Final = respx.post( + "https://bedrock-runtime.us-west-2.amazonaws.com/model/us.anthropic.claude-haiku-4-5-20251001-v1:0/converse" + ).mock( + return_value=httpx.Response( + 200, + json={ + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "call_trade", + "name": "trade", + "input": { + "orders": [ + { + "action": "buy", + "asset": "BTC", + "amount": 0.1, + } + ] + }, + } + } + ], + } + }, + "stopReason": "tool_use", + "usage": {"inputTokens": 8, "outputTokens": 5, "totalTokens": 13}, + }, + ) + ) + + response: Final = litellm.completion( + model="bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0", + messages=[{"role": "user", "content": "Buy 0.1 BTC"}], + tools=[ + { + "type": "function", + "function": { + "name": "trade", + "parameters": { + "type": "object", + "properties": { + "orders": { + "type": "array", + "items": { + "type": "object", + "properties": { + "action": {"type": "string"}, + "asset": {"type": "string"}, + "amount": {"type": "number"}, + }, + "required": ["action", "asset", "amount"], + }, + } + }, + "required": ["orders"], + }, + }, + } + ], + tool_choice={"type": "function", "function": {"name": "trade"}}, + api_key="test-bedrock-token", + aws_region_name="us-west-2", + ) + + tool_call: Final = response.choices[0].message.tool_calls[0] + assert route.call_count == 1 + assert tool_call.function.name == "trade" + assert json.loads(tool_call.function.arguments) == { + "orders": [{"action": "buy", "asset": "BTC", "amount": 0.1}] + } + + +@pytest.mark.parametrize( + ("model", "invoke_model_id", "response_body", "expected_request_body"), + [ + ( + "bedrock/mistral.mistral-7b-instruct-v0:2", + "mistral.mistral-7b-instruct-v0:2", + {"outputs": [{"text": "Going well, thanks!", "stop_reason": "stop"}]}, + {"prompt": "[INST] Hey! how's it going? [/INST]\n", "temperature": 0.2, "max_tokens": 200}, + ), + ( + "bedrock/invoke/meta.llama3-8b-instruct-v1:0", + "meta.llama3-8b-instruct-v1:0", + { + "generation": "Going well, thanks!", + "prompt_token_count": 12, + "generation_token_count": 5, + "stop_reason": "stop", + }, + { + "prompt": "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\nHey! how's it going?<|eot_id|>" + "<|start_header_id|>assistant<|end_header_id|>\n\n", + "temperature": 0.2, + "max_gen_len": 200, + }, + ), + ], +) +@respx.mock +def test_bedrock_invoke_signs_request_and_parses_scripted_response( + model: str, + invoke_model_id: str, + response_body: dict[str, object], + expected_request_body: dict[str, object], + fake_provider_credentials: None, +): + route: Final = respx.post( + f"https://bedrock-runtime.us-west-2.amazonaws.com/model/{invoke_model_id}/invoke" + ).mock(return_value=httpx.Response(200, json=response_body)) + + response: Final = litellm.completion( + model=model, + messages=[{"role": "user", "content": "Hey! how's it going?"}], + temperature=0.2, + max_tokens=200, + aws_region_name="us-west-2", + ) + + request: Final = route.calls[0].request + assert request.headers["authorization"].startswith( + "AWS4-HMAC-SHA256 Credential=unit-test/" + ) + assert "/us-west-2/bedrock/aws4_request" in request.headers["authorization"] + assert request.headers["x-amz-date"].endswith("Z") + assert json.loads(request.content) == expected_request_body + assert response.choices[0].message.content == "Going well, thanks!" + assert response.choices[0].finish_reason == "stop" + + +@respx.mock +def test_completion_azure_ad_token_is_forwarded(openai_api_response): + route: Final = respx.post( + "https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions" + ).mock(return_value=httpx.Response(200, json=openai_api_response)) + + litellm.completion( + model="azure/gpt-6.1-sol", + messages=[{"role": "user", "content": "hello"}], + api_base="https://example.openai.azure.com", + api_version="2023-07-01-preview", + azure_ad_token="my-special-token", + ) + + assert route.calls[0].request.headers["Authorization"] == "Bearer my-special-token" + + +@respx.mock +def test_completion_azure_extra_headers_are_forwarded(openai_api_response): + route: Final = respx.post( + "https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions" + ).mock(return_value=httpx.Response(200, json=openai_api_response)) + + litellm.completion( + model="azure/gpt-6.1-sol", + messages=[{"role": "user", "content": "hello"}], + api_base="https://example.openai.azure.com", + api_version="2023-07-01-preview", + api_key="azure-test-key", + extra_headers={ + "Authorization": "my-bad-key", + "Ocp-Apim-Subscription-Key": "test-subscription-key", + }, + ) + + request: Final = route.calls[0].request + assert request.headers["Authorization"] == "my-bad-key" + assert request.headers["Ocp-Apim-Subscription-Key"] == "test-subscription-key" + + +@respx.mock +def test_completion_azure_api_key_argument_is_forwarded(openai_api_response): + route: Final = respx.post( + "https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions" + ).mock(return_value=httpx.Response(200, json=openai_api_response)) + + response: Final = litellm.completion( + model="azure/gpt-6.1-sol", + messages=[{"role": "user", "content": "hello"}], + api_base="https://example.openai.azure.com", + api_version="2023-07-01-preview", + api_key="azure-test-key", + ) + + assert route.calls[0].request.headers["api-key"] == "azure-test-key" + assert response._hidden_params["custom_llm_provider"] == "azure" + + +@respx.mock +def test_completion_forwards_openai_optional_parameters(openai_api_response): + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, json=openai_api_response) + ) + messages: Final = [{"role": "user", "content": "return json"}] + + litellm.completion( + model="gpt-6-sol", + messages=messages, + temperature=0.25, + top_p=0.5, + seed=12, + max_tokens=10, + user="test-user", + response_format={"type": "json_object"}, + reasoning_effort="none", + logit_bias=None, + ) + + request_body: Final = json.loads(route.calls[0].request.content) + assert request_body == { + "model": "gpt-6-sol", + "messages": messages, + "temperature": 0.25, + "top_p": 0.5, + "seed": 12, + "max_completion_tokens": 10, + "user": "test-user", + "response_format": {"type": "json_object"}, + "reasoning_effort": "none", + } + + +@respx.mock +def test_completion_drops_unsupported_o3_mini_temperature(openai_api_response): + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, json=openai_api_response) + ) + + litellm.completion( + model="o3-mini", + messages=[{"role": "user", "content": "hello"}], + temperature=0.0, + drop_params=True, + ) + + request_body: Final = json.loads(route.calls[0].request.content) + assert request_body == { + "model": "o3-mini", + "messages": [{"role": "user", "content": "hello"}], + } + + +@pytest.mark.asyncio +@respx.mock +async def test_anthropic_tool_result_after_tool_call_is_translated(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + response_data: Final = { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [ + { + "type": "tool_use", + "id": "toolu_test", + "name": "submitFruit", + "input": {"name": "Apple"}, + } + ], + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 16, "output_tokens": 8}, + } + route: Final = respx.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response(200, json=response_data) + ) + messages: Final = [ + {"role": "system", "content": "Use the submitFruit function for a fruit."}, + {"role": "user", "content": "I like apples"}, + { + "role": "assistant", + "content": "Use the submitFruit function.", + "tool_calls": [ + { + "id": "toolu_test", + "type": "function", + "function": {"name": "submitFruit", "arguments": '{"name": "Apple"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "toolu_test", "content": '{"success":true}'}, + ] + tools: Final = [ + { + "type": "function", + "function": { + "name": "submitFruit", + "description": "Submits a fruit", + "parameters": { + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + }, + }, + } + ] + + response: Final = await litellm.acompletion( + model="anthropic/claude-sonnet-5-5", + messages=messages, + tools=tools, + max_tokens=128, + api_key="anthropic-test-key", + ) + + request_body: Final = json.loads(route.calls[0].request.content) + assert request_body["messages"][1]["content"] == [ + {"type": "text", "text": "Use the submitFruit function."}, + { + "type": "tool_use", + "id": "toolu_test", + "name": "submitFruit", + "input": {"name": "Apple"}, + }, + ] + assert request_body["messages"][2]["content"][0]["type"] == "tool_result" + assert request_body["messages"][2]["content"][0]["tool_use_id"] == "toolu_test" + assert response.choices[0].message.tool_calls[0].function.name == "submitFruit" + assert json.loads(response.choices[0].message.tool_calls[0].function.arguments) == {"name": "Apple"} + + +@respx.mock +def test_gemini_completion_translates_messages_and_parses_candidate(): + response_data: Final = { + "candidates": [ + { + "content": {"parts": [{"text": "Hello there!"}], "role": "model"}, + "finishReason": "STOP", + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 2, + "candidatesTokenCount": 2, + "totalTokenCount": 4, + }, + "modelVersion": "gemini-3.8-flash", + } + route: Final = respx.post( + "https://generativelanguage.googleapis.com/v1alpha/models/gemini-3.8-flash:generateContent" + ).mock(return_value=httpx.Response(200, json=response_data)) + + safety_settings: Final = [ + {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"}, + {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"}, + ] + response: Final = litellm.completion( + model="gemini/gemini-3.8-flash", + messages=[ + {"role": "system", "content": "Be a good bot!"}, + {"role": "user", "content": "Hey, how's it going?"}, + ], + safety_settings=safety_settings, + api_key="test-gemini-key", + ) + + request: Final = route.calls[0].request + request_body: Final = json.loads(request.content) + assert request.headers["x-goog-api-key"] == "test-gemini-key" + assert request_body["system_instruction"] == {"parts": [{"text": "Be a good bot!"}]} + assert request_body["contents"] == [{"parts": [{"text": "Hey, how's it going?"}], "role": "user"}] + assert request_body["safetySettings"] == safety_settings + assert response.choices[0].message.content == "Hello there!" + assert response.usage.total_tokens == 4 + + +@respx.mock +def test_qwen_text_completion_parses_text_and_logprobs(): + response_data: Final = { + "id": "cmpl-test", + "object": "text_completion", + "created": 1, + "model": "gpt-6-sol", + "choices": [ + { + "text": "hello", + "index": 0, + "logprobs": { + "tokens": ["hello"], + "token_logprobs": [-0.1], + "top_logprobs": [{"hello": -0.1}], + "text_offset": [0], + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + route: Final = respx.post("https://api.openai.com/v1/completions").mock( + return_value=httpx.Response(200, json=response_data) + ) + + response: Final = litellm.completion( + model="text-completion-openai/gpt-6-sol", + messages=[{"role": "user", "content": "hello"}], + logprobs=1, + ) + + assert len(route.calls) == 1 + request_body: Final = json.loads(route.calls[0].request.content) + assert request_body == {"model": "gpt-6-sol", "prompt": "hello", "logprobs": 1} + assert response.choices[0].message.content == "hello" + assert response.choices[0].logprobs.token_logprobs == [-0.1] + + +@respx.mock +def test_replicate_completion_applies_registered_prompt_template(monkeypatch: pytest.MonkeyPatch): + model: Final = "replicate/meta/llama-3-70b-instruct" + prompt: Final = "You are a good assistant[INST] What is 2 + 2? [/INST]Now answer as best you can:" + create_prediction: Final = respx.post( + "https://api.replicate.com/v1/models/meta/llama-3-70b-instruct/predictions" + ).mock( + return_value=httpx.Response( + 200, + json={ + "id": "prediction-test", + "urls": { + "get": "https://api.replicate.com/v1/predictions/prediction-test", + "cancel": "https://api.replicate.com/v1/predictions/prediction-test/cancel", + }, + }, + ) + ) + respx.get("https://api.replicate.com/v1/predictions/prediction-test").mock( + return_value=httpx.Response(200, json={"status": "succeeded", "output": ["4"]}) + ) + monkeypatch.setattr(litellm, "custom_prompt_dict", dict(litellm.custom_prompt_dict)) + litellm.register_prompt_template( + model=model, + initial_prompt_value="You are a good assistant", + final_prompt_value="Now answer as best you can:", + roles={ + "user": {"pre_message": "[INST] ", "post_message": " [/INST]"}, + }, + ) + + response: Final = litellm.completion( + model=model, + messages=[{"role": "user", "content": "What is 2 + 2?"}], + api_key="replicate-test-key", + ) + + request_body: Final = json.loads(create_prediction.calls[0].request.content) + assert request_body["input"] == {"prompt": prompt} + assert response.choices[0].message.content == "4" + + +@respx.mock +def test_petals_completion_sends_model_and_messages_to_custom_base(): + route: Final = respx.post("https://api.petals.dev/").mock( + return_value=httpx.Response(200, json={"outputs": "Hello!"}) + ) + messages: Final = [{"role": "user", "content": "hello"}] + + response: Final = litellm.completion( + model="petals-team/StableBeluga2", + messages=messages, + api_base="https://api.petals.dev", + max_tokens=7, + ) + + request: Final = route.calls[0].request + assert str(request.url) == "https://api.petals.dev/" + assert request.content == b"model=petals-team%2FStableBeluga2&inputs=hello&max_new_tokens=7" + assert response.choices[0].message.content == "Hello!" + + +@respx.mock +def test_petals_config_fills_unset_params_while_explicit_params_win(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm.PetalsConfig, "temperature", 0.4) + monkeypatch.setattr(litellm.PetalsConfig, "top_k", 5) + route: Final = respx.post("https://api.petals.dev/").mock( + return_value=httpx.Response(200, json={"outputs": "Hello!"}) + ) + messages: Final = [{"role": "user", "content": "hello"}] + + litellm.completion(model="petals-team/StableBeluga2", messages=messages, api_base="https://api.petals.dev") + litellm.completion( + model="petals-team/StableBeluga2", + messages=messages, + api_base="https://api.petals.dev", + max_tokens=7, + temperature=0.9, + ) + + assert [urllib.parse.parse_qs(call.request.content.decode()) for call in route.calls] == [ + { + "model": ["petals-team/StableBeluga2"], + "inputs": ["hello"], + "max_new_tokens": [str(litellm.max_tokens)], + "temperature": ["0.4"], + "top_k": ["5"], + }, + { + "model": ["petals-team/StableBeluga2"], + "inputs": ["hello"], + "max_new_tokens": ["7"], + "temperature": ["0.9"], + "top_k": ["5"], + }, + ] + + +@respx.mock +def test_moderation_posts_input_to_custom_base_and_parses_results(): + route: Final = respx.post("https://moderation.unit-test/v1/moderations").mock( + return_value=httpx.Response( + 200, + json={ + "id": "modr-unit-test", + "model": "omni-moderation-latest", + "results": [ + { + "flagged": True, + "categories": {"harassment": True, "violence": False}, + "category_scores": {"harassment": 0.91, "violence": 0.02}, + } + ], + }, + ) + ) + + response: Final = litellm.moderation( + input="i'm ishaan cto of litellm", + model="omni-moderation-latest", + api_key="sk-moderation-test", + api_base="https://moderation.unit-test/v1", + ) + + request: Final = route.calls[0].request + assert request.headers["authorization"] == "Bearer sk-moderation-test" + assert json.loads(request.content) == {"input": "i'm ishaan cto of litellm", "model": "omni-moderation-latest"} + assert response.id == "modr-unit-test" + assert response.results[0].flagged is True + assert (response.results[0].categories["harassment"], response.results[0].categories["violence"]) == (True, False) + assert ( + response.results[0].category_scores["harassment"], + response.results[0].category_scores["violence"], + ) == (0.91, 0.02) + + +@respx.mock +def test_mistral_tool_use_round_trip(): + first_response: Final = { + "id": "chatcmpl-tool", + "object": "chat.completion", + "created": 1, + "model": "mistral-medium-latest", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_weather", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": '{"location":"Boston","unit":"fahrenheit"}', + }, + } + ], + }, + } + ], + "usage": {"prompt_tokens": 12, "completion_tokens": 6, "total_tokens": 18}, + } + final_response: Final = { + "id": "chatcmpl-final", + "object": "chat.completion", + "created": 1, + "model": "mistral-large-latest", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "A sunny day."}, + } + ], + "usage": {"prompt_tokens": 30, "completion_tokens": 4, "total_tokens": 34}, + } + + def respond(request: httpx.Request) -> httpx.Response: + request_body: Final = json.loads(request.content) + return httpx.Response( + 200, + json=first_response if request_body["model"] == "mistral-medium-latest" else final_response, + ) + + route: Final = respx.post("https://api.mistral.ai/v1/chat/completions").mock(side_effect=respond) + messages: Final = [{"role": "user", "content": "What's the weather in Boston?"}] + tools: Final = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } + ] + tool_response: Final = litellm.completion( + model="mistral/mistral-medium-latest", + messages=messages, + tools=tools, + tool_choice="auto", + api_key="test-mistral-key", + ) + tool_call: Final = tool_response.choices[0].message.tool_calls[0] + assert tool_call.function.name == "get_current_weather" + assert tool_call.function.arguments == '{"location":"Boston","unit":"fahrenheit"}' + followup_messages: Final = [ + *messages, + tool_response.choices[0].message.model_dump(exclude_none=True), + { + "role": "tool", + "tool_call_id": tool_call.id, + "name": tool_call.function.name, + "content": '{"temperature":"72","unit":"fahrenheit"}', + }, + ] + final: Final = litellm.completion( + model="mistral/mistral-large-latest", + messages=followup_messages, + tools=tools, + tool_choice="auto", + api_key="test-mistral-key", + ) + + assert len(route.calls) == 2 + second_request: Final = json.loads(route.calls[1].request.content) + assert second_request["messages"][-1]["role"] == "tool" + assert second_request["messages"][-1]["tool_call_id"] == "call_weather" + assert final.choices[0].message.content == "A sunny day." + + +@respx.mock +def test_mistral_text_content_array_is_sent_as_text(): + response_data: Final = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "mistral-large-latest", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Hello."}, + } + ], + "usage": {"prompt_tokens": 8, "completion_tokens": 2, "total_tokens": 10}, + } + route: Final = respx.post("https://api.mistral.ai/v1/chat/completions").mock( + return_value=httpx.Response(200, json=response_data) + ) + + response: Final = litellm.completion( + model="mistral/mistral-large-latest", + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "Hey, how's it going?"}], + } + ], + api_key="test-mistral-key", + input_cost_per_token=0.0000008, + output_cost_per_token=0.0000032, + ) + + request_body: Final = json.loads(route.calls[0].request.content) + assert request_body == { + "model": "mistral-large-latest", + "messages": [{"role": "user", "content": "Hey, how's it going?"}], + } + assert response._hidden_params["response_cost"] == pytest.approx(8 * 0.0000008 + 2 * 0.0000032) + + +def test_mock_request_returns_requested_response(): + response: Final = litellm.mock_completion( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + mock_response="mocked answer", + ) + + assert response.choices[0].message.content == "mocked answer" + + +def test_mock_request_with_mock_timeout_raises_litellm_timeout(): + with pytest.raises(litellm.Timeout) as error: + litellm.completion( + model="gpt-5.6", + messages=[{"role": "user", "content": "hello"}], + timeout=0.01, + mock_timeout=True, + num_retries=0, + ) + + assert error.value.model == "gpt-5.6" + + +def test_router_mock_request_with_mock_timeout_raises_litellm_timeout(): + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gpt-5.6", + "litellm_params": { + "model": "gpt-5.6", + "api_key": "test-key", + }, + } + ], + num_retries=0, + ) + + with pytest.raises(litellm.Timeout) as error: + router.completion( + model="gpt-5.6", + messages=[{"role": "user", "content": "hello"}], + timeout=0.01, + mock_timeout=True, + ) + + assert error.value.model == "gpt-5.6" + + +@respx.mock +def test_router_mock_request_fallback_uses_fallback_model(openai_api_response): + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, json={**openai_api_response, "model": "gpt-5.5"}) + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gpt-5.6", + "litellm_params": {"model": "gpt-5.6", "api_key": "test-key"}, + }, + { + "model_name": "gpt-5.5", + "litellm_params": {"model": "gpt-5.5", "api_key": "test-key"}, + }, + ], + fallbacks=[{"gpt-5.6": ["gpt-5.5"]}], + num_retries=0, + ) + + response: Final = router.completion( + model="gpt-5.6", + messages=[{"role": "user", "content": "hello"}], + timeout=0.01, + mock_timeout=True, + ) + + assert [json.loads(call.request.content)["model"] for call in route.calls] == ["gpt-5.5"] + assert response.model == "gpt-5.5" + + @pytest.mark.parametrize("drop_params", [True, False]) +@respx.mock def test_completion_deep_infra(drop_params): - """Test that DeepInfra requests are shaped correctly without making real API calls.""" - from unittest.mock import MagicMock, patch - - import httpx - from openai.types.chat import ChatCompletion, ChatCompletionMessage - from openai.types.chat.chat_completion import Choice - litellm.set_verbose = False - model_name = "deepinfra/meta-llama/Llama-2-70b-chat-hf" - tools = [ + model_name: Final = "deepinfra/meta-llama/Llama-2-70b-chat-hf" + model: Final = "meta-llama/Llama-2-70b-chat-hf" + endpoint: Final = "https://api.deepinfra.com/v1/openai/chat/completions" + tools: Final = [ { "type": "function", "function": { @@ -5332,131 +6719,114 @@ def test_completion_deep_infra(drop_params): }, } ] - messages = [ + messages: Final = [ { "role": "user", "content": "What's the weather like in Boston today in Fahrenheit?", } ] - - mock_response = ChatCompletion( - id="chatcmpl-mock", - choices=[ - Choice( - finish_reason="stop", - index=0, - message=ChatCompletionMessage(content="It's sunny.", role="assistant"), - ) + response_body: Final = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1234567890, + "model": model, + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "It's sunny."}, + } ], - created=1234567890, - model="meta-llama/Llama-2-70b-chat-hf", - object="chat.completion", - usage={"completion_tokens": 5, "prompt_tokens": 20, "total_tokens": 25}, + "usage": {"prompt_tokens": 20, "completion_tokens": 5, "total_tokens": 25}, + } + route: Final = respx.post(endpoint).mock( + return_value=httpx.Response(200, json=response_body) ) - mock_raw = MagicMock() - mock_raw.parse.return_value = mock_response - mock_raw.headers = httpx.Headers({"content-type": "application/json"}) - mock_raw.status_code = 200 + if not drop_params: + with pytest.raises(litellm.exceptions.UnsupportedParamsError): + completion( + model=model_name, + messages=messages, + temperature=0, + max_tokens=10, + tools=tools, + tool_choice={"type": "function", "function": {"name": "get_current_weather"}}, + drop_params=False, + api_key="fake-api-key", + ) + assert route.calls == [] + return - with patch( - "litellm.llms.openai.openai.OpenAIChatCompletion.make_sync_openai_chat_completion_request", - return_value=(mock_raw, mock_response), - ) as mock_create: - if drop_params is False: - # DeepInfra doesn't support tool_choice, should raise UnsupportedParamsError - with pytest.raises(litellm.exceptions.UnsupportedParamsError): - completion( - model=model_name, - messages=messages, - temperature=0, - max_tokens=10, - tools=tools, - tool_choice={ - "type": "function", - "function": {"name": "get_current_weather"}, - }, - drop_params=drop_params, - api_key="fake-api-key", - ) - return + response: Final = completion( + model=model_name, + messages=messages, + temperature=0, + max_tokens=10, + tools=tools, + tool_choice={ + "type": "function", + "function": {"name": "get_current_weather"}, + }, + drop_params=drop_params, + api_key="fake-api-key", + ) - response = completion( - model=model_name, - messages=messages, - temperature=0, - max_tokens=10, - tools=tools, - tool_choice={ - "type": "function", - "function": {"name": "get_current_weather"}, - }, - drop_params=drop_params, - api_key="fake-api-key", - ) - - # Verify the call was made - mock_create.assert_called_once() - call_kwargs = mock_create.call_args.kwargs - - # Verify request shape - data = call_kwargs["data"] - assert data["model"] == "meta-llama/Llama-2-70b-chat-hf" - assert data["messages"] == messages - assert data["temperature"] == 0 - assert data["max_tokens"] == 10 - # tool_choice should be dropped for unsupported params - assert "tool_choice" not in data + assert route.call_count == 1 + request_body: Final = json.loads(route.calls.last.request.read()) + assert request_body == { + "model": model, + "messages": messages, + "temperature": 0, + "max_tokens": 10, + "tools": tools, + } + assert response.choices[0].message.content == "It's sunny." + assert response.choices[0].finish_reason == "stop" +@respx.mock def test_completion_deep_infra_mistral(): - """Test that DeepInfra Mistral requests are shaped correctly without making real API calls.""" - from unittest.mock import MagicMock, patch - - import httpx - from openai.types.chat import ChatCompletion, ChatCompletionMessage - from openai.types.chat.chat_completion import Choice - - model_name = "deepinfra/mistralai/Mistral-7B-Instruct-v0.1" - - mock_response = ChatCompletion( - id="chatcmpl-mock", - choices=[ - Choice( - finish_reason="stop", - index=0, - message=ChatCompletionMessage(content="Hello!", role="assistant"), - ) + model_name: Final = "deepinfra/mistralai/Mistral-7B-Instruct-v0.1" + model: Final = "mistralai/Mistral-7B-Instruct-v0.1" + messages: Final = [{"role": "user", "content": "Say hello."}] + endpoint: Final = "https://api.deepinfra.com/v1/openai/chat/completions" + response_body: Final = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1234567890, + "model": model, + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Hello!"}, + } ], - created=1234567890, - model="mistralai/Mistral-7B-Instruct-v0.1", - object="chat.completion", - usage={"completion_tokens": 5, "prompt_tokens": 20, "total_tokens": 25}, + "usage": {"prompt_tokens": 20, "completion_tokens": 5, "total_tokens": 25}, + } + route: Final = respx.post(endpoint).mock( + return_value=httpx.Response(200, json=response_body) ) - mock_raw = MagicMock() - mock_raw.parse.return_value = mock_response - mock_raw.headers = httpx.Headers({"content-type": "application/json"}) - mock_raw.status_code = 200 + response: Final = completion( + model=model_name, + messages=messages, + temperature=0.01, + max_tokens=10, + api_key="fake-api-key", + ) - with patch( - "litellm.llms.openai.openai.OpenAIChatCompletion.make_sync_openai_chat_completion_request", - return_value=(mock_raw, mock_response), - ) as mock_create: - response = completion( - model=model_name, - messages=messages, - temperature=0.01, - max_tokens=10, - api_key="fake-api-key", - ) - - mock_create.assert_called_once() - call_kwargs = mock_create.call_args.kwargs - data = call_kwargs["data"] - assert data["model"] == "mistralai/Mistral-7B-Instruct-v0.1" - assert data["temperature"] == 0.01 - assert data["max_tokens"] == 10 + assert route.call_count == 1 + request_body: Final = json.loads(route.calls.last.request.read()) + assert request_body == { + "model": model, + "messages": messages, + "temperature": 0.01, + "max_tokens": 10, + } + assert response.choices[0].message.content == "Hello!" + assert response.choices[0].finish_reason == "stop" @pytest.mark.parametrize( @@ -6188,6 +7558,580 @@ def test_azure_embedding_exceptions(): assert str(exc_info.value) == "Mock error" +@respx.mock +def test_openai_chat_stream_routes_legacy_function_calls_and_usage( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "return_response_headers", True) + api_base: Final = "https://openai-unit.test/v1" + messages: Final = ({"role": "user", "content": "What is the weather in SF?"},) + functions: Final = ( + { + "name": "get_current_weather", + "description": "Get current weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, + ) + answer_events: Final = ( + { + "id": "chatcmpl-answer", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "The answer is "}, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-answer", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "delta": {"content": "42."}, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-answer", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + { + "id": "chatcmpl-answer", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [], + "usage": {"prompt_tokens": 4, "completion_tokens": 2, "total_tokens": 6}, + }, + ) + function_events: Final = ( + { + "id": "chatcmpl-function", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "function_call": { + "name": "get_current_weather", + "arguments": '{"location":', + }, + }, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-function", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "delta": {"function_call": {"arguments": '"San Francisco"}'}}, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-function", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "delta": {}, "finish_reason": "function_call"} + ], + }, + ) + + def response_body(events: tuple[dict[str, object], ...]) -> str: + return "".join(f"data: {json.dumps(event)}\n\n" for event in events) + "data: [DONE]\n\n" + + scripted_responses: Final = iter( + ( + httpx.Response( + 200, + text=response_body(answer_events), + headers={ + "content-type": "text/event-stream", + "x-ratelimit-remaining-requests": "17", + }, + ), + httpx.Response( + 200, + text=response_body(function_events), + headers={"content-type": "text/event-stream"}, + ), + ) + ) + route: Final = respx.post(f"{api_base}/chat/completions").mock( + side_effect=lambda _: next(scripted_responses) + ) + stream: Final = litellm.completion( + model="gpt-4o-mini", + api_key="sk-test", + api_base=api_base, + messages=messages, + stream=True, + stream_options={"include_usage": True}, + ) + answer_chunks: Final = tuple(stream) + + assert "".join( + chunk.choices[0].delta.content + for chunk in answer_chunks + if chunk.choices and chunk.choices[0].delta.content + ) == "The answer is 42." + assert tuple( + chunk.choices[0].finish_reason + for chunk in answer_chunks + if chunk.choices and chunk.choices[0].finish_reason is not None + ) == ("stop",) + assert answer_chunks[-2].choices[0].finish_reason == "stop" + assert tuple(getattr(chunk, "usage", None) for chunk in answer_chunks[:-1]) == ( + None, + None, + None, + ) + assert answer_chunks[-1].usage is not None + assert answer_chunks[-1].usage.prompt_tokens == 4 + assert answer_chunks[-1].usage.completion_tokens == 2 + assert answer_chunks[-1].usage.total_tokens == 6 + assert stream._hidden_params["api_base"] == api_base + assert ( + answer_chunks[0]._hidden_params["additional_headers"][ + "llm_provider-x-ratelimit-remaining-requests" + ] + == "17" + ) + + function_stream: Final = litellm.completion( + model="gpt-4o-mini", + api_key="sk-test", + api_base=api_base, + messages=messages, + functions=functions, + function_call={"name": "get_current_weather"}, + stream=True, + ) + function_chunks: Final = tuple(function_stream) + assembled: Final = litellm.stream_chunk_builder( + chunks=function_chunks, messages=messages + ) + function_call: Final = assembled.choices[0].message.function_call + + assert function_call is not None + assert function_call.name == "get_current_weather" + assert json.loads(function_call.arguments) == {"location": "San Francisco"} + assert tuple( + chunk.choices[0].finish_reason + for chunk in function_chunks + if chunk.choices and chunk.choices[0].finish_reason is not None + ) == ("function_call",) + assert route.call_count == 2 + assert json.loads(route.calls[0].request.content)["stream_options"] == { + "include_usage": True + } + assert json.loads(route.calls[1].request.content)["function_call"] == { + "name": "get_current_weather" + } + assert json.loads(route.calls[1].request.content)["functions"] == list(functions) + + +@respx.mock +async def test_async_openai_stream_options_include_usage_on_the_last_chunk( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + events: Final = ( + { + "id": "chatcmpl-async-usage", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "async reply"}, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-async-usage", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + { + "id": "chatcmpl-async-usage", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, + }, + ) + body: Final = "".join( + f"data: {json.dumps(event)}\n\n" for event in events + ) + "data: [DONE]\n\n" + route: Final = respx.post( + "https://api.openai.com/v1/chat/completions" + ).mock( + return_value=httpx.Response( + 200, text=body, headers={"content-type": "text/event-stream"} + ) + ) + stream: Final = await litellm.acompletion( + model="gpt-4o-mini", + api_key="sk-test", + messages=[{"role": "user", "content": "say hello"}], + stream=True, + stream_options={"include_usage": True}, + ) + chunks: Final = tuple([chunk async for chunk in stream]) + + assert "".join( + chunk.choices[0].delta.content + for chunk in chunks + if chunk.choices and chunk.choices[0].delta.content + ) == "async reply" + assert tuple(getattr(chunk, "usage", None) for chunk in chunks[:-1]) == ( + None, + None, + ) + assert chunks[-1].usage.prompt_tokens == 3 + assert chunks[-1].usage.completion_tokens == 2 + assert chunks[-1].usage.total_tokens == 5 + assert json.loads(route.calls[0].request.content)["stream_options"] == { + "include_usage": True + } + + +@respx.mock +async def test_parallel_openai_streams_keep_prompt_responses_separate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + def respond(request: httpx.Request) -> httpx.Response: + prompt: Final = json.loads(request.content)["messages"][0]["content"] + response_text: Final = { + "request one": "response one", + "request two": "response two", + }[prompt] + events: Final = ( + { + "id": "chatcmpl-parallel", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": response_text}, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-parallel", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + ) + body: Final = "".join( + f"data: {json.dumps(event)}\n\n" for event in events + ) + "data: [DONE]\n\n" + return httpx.Response( + 200, text=body, headers={"content-type": "text/event-stream"} + ) + + route: Final = respx.post( + "https://api.openai.com/v1/chat/completions" + ).mock(side_effect=respond) + + async def collect(prompt: str) -> tuple[str, tuple[str, ...]]: + stream = await litellm.acompletion( + model="gpt-4o-mini", + api_key="sk-test", + messages=[{"role": "user", "content": prompt}], + stream=True, + ) + chunks: Final = tuple([chunk async for chunk in stream]) + response_text: Final = "".join( + chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices + ) + finish_reasons: Final = tuple( + chunk.choices[0].finish_reason + for chunk in chunks + if chunk.choices and chunk.choices[0].finish_reason is not None + ) + return response_text, finish_reasons + + responses: Final = tuple( + await asyncio.gather(collect("request one"), collect("request two")) + ) + + assert tuple(response for response, _ in responses) == ( + "response one", + "response two", + ) + assert tuple(finish_reasons for _, finish_reasons in responses) == ( + ("stop",), + ("stop",), + ) + assert route.call_count == 2 + + +@respx.mock +def test_parallel_sync_openai_streams_in_threads_keep_prompt_responses_separate() -> None: + both_requests_in_flight: Final = threading.Barrier(2, timeout=10) + + def respond(request: httpx.Request) -> httpx.Response: + prompt: Final = json.loads(request.content)["messages"][0]["content"] + both_requests_in_flight.wait() + events: Final = ( + { + "id": "chatcmpl-parallel-sync", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": prompt.replace("request", "response")}, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-parallel-sync", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + ) + body: Final = "".join(f"data: {json.dumps(event)}\n\n" for event in events) + "data: [DONE]\n\n" + return httpx.Response(200, text=body, headers={"content-type": "text/event-stream"}) + + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock(side_effect=respond) + + def collect(prompt: str) -> tuple[str, tuple[str, ...]]: + stream: Final = litellm.completion( + model="gpt-4o-mini", + api_key="sk-test", + messages=[{"role": "user", "content": prompt}], + stream=True, + ) + chunks: Final = tuple(stream) + return ( + "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices), + tuple( + chunk.choices[0].finish_reason + for chunk in chunks + if chunk.choices and chunk.choices[0].finish_reason is not None + ), + ) + + with ThreadPoolExecutor(max_workers=2) as pool: + responses: Final = tuple(pool.map(collect, ("request one", "request two"))) + + assert responses == (("response one", ("stop",)), ("response two", ("stop",))) + assert route.call_count == 2 + + +@respx.mock +def test_openai_streaming_tool_call_arguments_are_valid_json( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + chunks: Final = ( + { + "id": "chatcmpl-tools", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "delta": { + "tool_calls": [ + { + "index": 0, + "id": "call_weather", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": '{"location":"Boston",', + }, + } + ] + }, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-tools", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "delta": { + "tool_calls": [ + { + "index": 0, + "function": {"arguments": '"unit":"fahrenheit"}'}, + } + ] + }, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-tools", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}], + }, + ) + body: Final = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n" + respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + text=body, + headers={"content-type": "text/event-stream"}, + ) + ) + messages: Final = [{"role": "user", "content": "What is the weather in Boston?"}] + stream: Final = litellm.completion( + model="gpt-4o-mini", + messages=messages, + tools=[ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + "unit": {"type": "string"}, + }, + "required": ["location", "unit"], + }, + }, + } + ], + stream=True, + api_key="test-openai-key", + ) + stream_chunks: Final = tuple(stream) + assembled: Final = litellm.stream_chunk_builder(chunks=list(stream_chunks), messages=messages) + assert assembled is not None + assert assembled.choices[0].message.tool_calls[0].function.name == "get_current_weather" + assert json.loads(assembled.choices[0].message.tool_calls[0].function.arguments) == { + "location": "Boston", + "unit": "fahrenheit", + } + + +@respx.mock +def test_streaming_openai_usage_includes_test_owned_cost( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + model_name: Final = "stream-cost-test-model" + monkeypatch.setitem( + litellm.model_cost, + model_name, + { + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + "litellm_provider": "openai", + "mode": "chat", + }, + ) + chunks: Final = ( + { + "id": "chatcmpl-cost", + "object": "chat.completion.chunk", + "created": 1, + "model": model_name, + "choices": [ + {"index": 0, "delta": {"role": "assistant", "content": "priced"}, "finish_reason": None} + ], + }, + { + "id": "chatcmpl-cost", + "object": "chat.completion.chunk", + "created": 1, + "model": model_name, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + { + "id": "chatcmpl-cost", + "object": "chat.completion.chunk", + "created": 1, + "model": model_name, + "choices": [], + "usage": {"prompt_tokens": 4, "completion_tokens": 2, "total_tokens": 6}, + }, + ) + body: Final = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n" + respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + text=body, + headers={"content-type": "text/event-stream"}, + ) + ) + stream: Final = litellm.completion( + model=f"openai/{model_name}", + messages=[{"role": "user", "content": "price the response"}], + stream=True, + stream_options={"include_usage": True}, + api_key="test-openai-key", + ) + stream_chunks: Final = tuple(stream) + usage_chunks: Final = tuple( + getattr(chunk, "usage", None) + for chunk in stream_chunks + if getattr(chunk, "usage", None) is not None + ) + assert len(usage_chunks) == 1 + assert usage_chunks[0].cost == pytest.approx(4 * 0.001 + 2 * 0.002) + + def test_model_alias_map_resolves_the_outbound_model( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch diff --git a/tests/unit/test_openai_embedding_encoding_format_default.py b/tests/unit/test_openai_embedding_encoding_format_default.py index 9788eb30b1b..8b1b451b0b0 100644 --- a/tests/unit/test_openai_embedding_encoding_format_default.py +++ b/tests/unit/test_openai_embedding_encoding_format_default.py @@ -6,6 +6,7 @@ import pytest import respx import litellm +from openai.types import CreateEmbeddingResponse _SDK_EMBEDDING_MODELS: Final = ( "openai/sdk-compat", @@ -24,7 +25,10 @@ def _mock_openai_embedding_route( 200, json={ "object": "list", - "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], + "data": [ + {"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}, + {"object": "embedding", "index": 1, "embedding": [0.4, 0.5, 0.6]}, + ], "model": "text-embedding-3-small", "usage": {"prompt_tokens": 2, "total_tokens": 2}, }, @@ -143,3 +147,109 @@ def test_embedding_openai_omitted_encoding_format_maps_provider_errors( ) assert int(exc_info.value.litellm_response_headers["retry-after"]) == 42 + + +@pytest.mark.parametrize( + ("model", "url", "extra_params"), + ( + ("openai/text-embedding-3-small", "https://api.openai.com/v1/embeddings", {}), + ( + "azure/migration-embedding-deployment", + "https://migration.openai.azure.com/openai/deployments/migration-embedding-deployment/embeddings", + {"api_base": "https://migration.openai.azure.com", "api_version": "2024-10-21"}, + ), + ), +) +def test_embedding_response_preserves_rate_limit_headers( + respx_mock: respx.MockRouter, model: str, url: str, extra_params: dict[str, str] +) -> None: + respx_mock.post(url__startswith=url).mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], + "model": "text-embedding-3-small", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + }, + headers={ + "x-ratelimit-limit-requests": "10", + "x-ratelimit-remaining-requests": "9", + "x-ratelimit-remaining-tokens": "99", + }, + ) + ) + + response: Final = litellm.embedding(model=model, input=["Hello world"], api_key="sk-test", **extra_params) + + additional_headers: Final = response._hidden_params["additional_headers"] + assert { + name: additional_headers[name] + for name in ( + "x-ratelimit-remaining-requests", + "x-ratelimit-remaining-tokens", + "llm_provider-x-ratelimit-remaining-requests", + "llm_provider-x-ratelimit-remaining-tokens", + ) + } == { + "x-ratelimit-remaining-requests": "9", + "x-ratelimit-remaining-tokens": "99", + "llm_provider-x-ratelimit-remaining-requests": "9", + "llm_provider-x-ratelimit-remaining-tokens": "99", + } + + +def _openai_sdk_response_keys(route: respx.Route) -> frozenset[str]: + return frozenset(dict(CreateEmbeddingResponse.model_validate(route.return_value.json()))) + + +def test_openai_embedding_returns_sdk_shaped_vectors(respx_mock: respx.MockRouter) -> None: + route: Final = _mock_openai_embedding_route(respx_mock) + + response: Final = litellm.embedding( + model="openai/text-embedding-3-small", + input=["first", "second"], + api_key="sk-test", + ) + + assert frozenset(dict(response)) - {"_response_ms"} == _openai_sdk_response_keys(route) + assert [row["embedding"] for row in response.data] == [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]] + assert len(route.calls) == 1 + + +def test_openai_embedding_forwards_dimensions(respx_mock: respx.MockRouter) -> None: + route: Final = _mock_openai_embedding_route(respx_mock) + + response: Final = litellm.embedding( + model="openai/text-embedding-3-small", + input=["first", "second"], + api_key="sk-test", + dimensions=5, + ) + + body: Final = json.loads(route.calls.last.request.read()) + assert body["dimensions"] == 5 + assert body["input"] == ["first", "second"] + assert frozenset(dict(response)) - {"_response_ms"} == _openai_sdk_response_keys(route) + assert len(response.data) == 2 + + +@pytest.mark.asyncio +async def test_aembedding_openai_returns_the_mocked_vector( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + _mock_openai_embedding_route(respx_mock) + + response: Final = await litellm.aembedding( + model="openai/text-embedding-3-small", + input=["good morning from litellm", "this is another item"], + api_key="sk-test", + ) + cost: Final = litellm.completion_cost( + completion_response=response, + custom_cost_per_token={"input_cost_per_token": 0.001, "output_cost_per_token": 0.0}, + ) + + assert [row["embedding"] for row in response.data] == [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]] + assert cost == pytest.approx(0.002) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 7a192d17347..6b63f69fd2e 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,4 +1,5 @@ import asyncio, importlib, re +import ast import base64 import copy import contextlib @@ -12,7 +13,7 @@ import threading from collections.abc import Callable, Iterator, Mapping from concurrent.futures import Future, ThreadPoolExecutor from datetime import datetime, timedelta, timezone -from pathlib import PurePath +from pathlib import Path, PurePath from typing import Final, cast from unittest import mock from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -2135,6 +2136,118 @@ class TestProxyFunctionCalling: ) +@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") +def test_register_model_price_is_used_for_completion_cost() -> None: + model: Final = "migration-custom-pricing-model" + price: Final = { + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + "litellm_provider": "openai", + "mode": "chat", + } + litellm.register_model({model: price}, persist_across_reloads=False) + response: Final = ModelResponse( + model=model, + choices=[], + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) + + cost: Final = litellm.completion_cost( + completion_response=response, + model=model, + custom_llm_provider="openai", + ) + + assert cost == pytest.approx(0.02) + + +@respx.mock +@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") +def test_completion_registers_test_owned_model_prices() -> None: + model: Final = "migration-owned-pricing-model" + messages: Final = [{"role": "user", "content": "price control"}] + litellm.register_model( + { + model: { + "input_cost_per_token": 0.0001, + "output_cost_per_token": 0.0002, + "litellm_provider": "openai", + "mode": "chat", + } + }, + persist_across_reloads=False, + ) + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-priced", + "object": "chat.completion", + "created": 123, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "response"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 8, "completion_tokens": 5, "total_tokens": 13}, + }, + ) + ) + + response: Final = litellm.completion( + model=model, + messages=messages, + api_key="test-api-key", + input_cost_per_token=0.001, + output_cost_per_token=0.002, + ) + + assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages} + assert litellm.model_cost[model]["input_cost_per_token"] == 0.001 + assert litellm.model_cost[model]["output_cost_per_token"] == 0.002 + assert response._hidden_params["response_cost"] == pytest.approx(0.018) + + +@pytest.mark.usefixtures("isolated_openai_model_sets") +def test_add_known_models_skips_region_scoped_bedrock_pricing_keys(monkeypatch: pytest.MonkeyPatch) -> None: + price: Final = {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002, "mode": "chat"} + model_cost_map: Final = { + "migration.bedrock-model-v1:0": {**price, "litellm_provider": "bedrock"}, + "bedrock/us-west-1/migration.bedrock-model-v1:0": {**price, "litellm_provider": "bedrock"}, + "migration-openai-model": {**price, "litellm_provider": "openai"}, + } + monkeypatch.setattr(litellm, "bedrock_models", set(litellm.bedrock_models)) + monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider)) + + litellm.add_known_models(model_cost_map=model_cost_map) + + assert "migration.bedrock-model-v1:0" in litellm.bedrock_models + assert "bedrock/us-west-1/migration.bedrock-model-v1:0" not in litellm.bedrock_models + assert "migration-openai-model" in litellm.models_by_provider["openai"] + + +def test_utils_defines_tests_without_invoking_them_at_module_scope() -> None: + tree: Final = ast.parse(Path(__file__).read_text()) + defined: Final = frozenset( + node.name + for node in tree.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + ) + invoked: Final = tuple( + node.value.func.id + for node in tree.body + if isinstance(node, ast.Expr) + and isinstance(node.value, ast.Call) + and isinstance(node.value.func, ast.Name) + and node.value.func.id in defined + ) + + assert invoked == () + + @pytest.mark.usefixtures("isolated_openai_model_sets") def test_register_model_with_scientific_notation(): """ @@ -9557,3 +9670,43 @@ async def test_responses_retry_on_auth_error(sync_mode, respx_mock: respx.MockRo assert mock_retry.called assert mock_retry.call_args.kwargs.get("num_retries") == num_retries + + +@respx.mock +def test_pre_call_rules_see_messages_passed_positionally(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "pre_call_rules", [lambda text: "forbidden" not in text]) + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-rules", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "ok"}}], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, + }, + ) + ) + + with pytest.raises(litellm.APIResponseValidationError, match="failed post-call-rule check"): + litellm.completion( + "gpt-4o-mini", [{"role": "user", "content": "say something forbidden"}], api_key="sk-test" + ) + allowed: Final = litellm.completion("gpt-4o-mini", [{"role": "user", "content": "say hi"}], api_key="sk-test") + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content)["messages"] == [{"role": "user", "content": "say hi"}] + assert allowed.choices[0].message.content == "ok" + + +def test_vertex_llama3_model_info_resolves_without_the_meta_prefix(): + llama3_models: Final = sorted(litellm.vertex_llama3_models) + assert llama3_models != [] + + for model in llama3_models: + short_name = model.removeprefix("meta/") + assert short_name != model + prefixed_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") + assert litellm.get_model_info(model=short_name, custom_llm_provider="vertex_ai") == prefixed_info, model + assert prefixed_info["key"] == f"vertex_ai/{model}", model