diff --git a/.github/actions/cache-cargo-build/action.yml b/.github/actions/cache-cargo-build/action.yml index c3b8ce22c68..222fad637fb 100644 --- a/.github/actions/cache-cargo-build/action.yml +++ b/.github/actions/cache-cargo-build/action.yml @@ -15,6 +15,12 @@ description: >- cache the same directory for different workloads, and a shared key would let whichever ran first deny the others a save. +inputs: + profile: + description: "Cargo profile the build uses (dev or release)" + required: false + default: "dev" + runs: using: composite steps: @@ -25,6 +31,6 @@ runs: ~/.cargo/registry ~/.cargo/git litellm-rust/target - key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }} + key: ${{ runner.os }}-maturin-${{ inputs.profile }}-${{ hashFiles('litellm-rust/Cargo.lock') }} restore-keys: | - ${{ runner.os }}-maturin-dev- + ${{ runner.os }}-maturin-${{ inputs.profile }}- diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py index 7913b25918b..691c10203bf 100644 --- a/.github/e2e-stack/secrets_to_env.py +++ b/.github/e2e-stack/secrets_to_env.py @@ -9,6 +9,7 @@ from pydantic import TypeAdapter, ValidationError secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str]) ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") MIN_MASKED_LENGTH: Final = 8 +ACTIONS_RUNNER_FLAG: Final = "GITHUB_ACTIONS" def main() -> int: @@ -30,10 +31,15 @@ def main() -> int: f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n" ) return 1 - for value in secrets.values(): - if len(value) >= MIN_MASKED_LENGTH: - _ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n") - sys.stdout.flush() + if os.environ.get(ACTIONS_RUNNER_FLAG) == "true": + _ = sys.stdout.write( + "".join( + f"::add-mask::{value.replace('%', '%25')}\n" + for value in secrets.values() + if len(value) >= MIN_MASKED_LENGTH + ) + ) + sys.stdout.flush() lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value) try: with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle: diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py index dbc4ae8f5c2..492c52233cd 100644 --- a/.github/e2e-stack/select_tests.py +++ b/.github/e2e-stack/select_tests.py @@ -9,6 +9,8 @@ UNSUPPORTED: Final = re.compile( r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$" r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$" r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$" + r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$" + r"|^tests/e2e/logging/test_langsmith_batch_serialization_e2e\.py$" ) HARNESS: Final = re.compile( r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$" diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh index 928b58e93bb..931b5eb2169 100755 --- a/.github/e2e-stack/up.sh +++ b/.github/e2e-stack/up.sh @@ -24,6 +24,7 @@ DATABASE_USER="${E2E_DATABASE_USER:-litellm}" DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}" DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}" JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}" +JAEGER_OTLP_TLS_PORT="${E2E_JAEGER_OTLP_TLS_PORT:-4319}" JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}" KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}" @@ -122,7 +123,7 @@ SERVER_ENV=( "CONFIG_FILE_PATH=${CONFIG_PATH}" "STORE_MODEL_IN_DB=True" "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf" - "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}" + "OTEL_EXPORTER_OTLP_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}" "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem" "PYTHONPATH=${REPO_ROOT}" "JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs" @@ -147,16 +148,12 @@ start_server() { echo $! > "${PIDS_DIR}/${name}.pid" } -start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}" -start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}" -start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}" - if [[ "$(uname)" == "Linux" ]]; then NGINX_UPSTREAM_HOST=127.0.0.1 NGINX_DOCKER_ARGS=(--network host) else NGINX_UPSTREAM_HOST=host.docker.internal - NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}") + NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}" -p "${JAEGER_OTLP_TLS_PORT}:${JAEGER_OTLP_TLS_PORT}") fi cat > "${STACK_DIR}/nginx.conf" </dev/null 2>&1 || true docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \ - -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" "${NGINX_IMAGE}" >/dev/null + -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" \ + -v "${CERTS_DIR}:/certs:ro" "${NGINX_IMAGE}" >/dev/null + +wait_for "Jaeger OTLP TLS listener" \ + "curl -sS --cacert ${CERTS_DIR}/ca.crt https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}/ -o /dev/null -w '%{http_code}' | grep -qE '^[2345]'" + +start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}" +start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}" +start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}" wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300 wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300 @@ -206,6 +220,7 @@ LITELLM_MASTER_KEY=${MASTER_KEY} REDIS_HOST=127.0.0.1 REDIS_PORT=${REDIS_PORT} E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT} +E2E_OTEL_EXPORTER_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT} E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT} E2E_KEYCLOAK_ADMIN_USER=admin E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py index ea6d2401084..f2b82f86b47 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/.github/scripts/verify_linux_native_wheel.py @@ -134,7 +134,16 @@ def main( uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members) native_path: Final = wheel.parent / "native" / Path(native_member.filename).name native_path.parent.mkdir(parents=True, exist_ok=True) - native_path.write_bytes(archive.read(native_member)) + native_bytes: Final = archive.read(native_member) + native_path.write_bytes(native_bytes) + duplicated_vocabularies: Final = tuple( + member.filename + for member in wheel_members + if member.filename.startswith("litellm/litellm_core_utils/tokenizers/") + and re.fullmatch(r"[0-9a-f]{40}", PurePosixPath(member.filename).name) + and member.file_size > 0 + and archive.read(member) in native_bytes + ) wheel_metadata_tags_match: Final = ( len(wheel_metadata_tags) == len(expanded_filename_tags) @@ -205,7 +214,7 @@ def main( native_module: Final = load_native_module(native_path) native_module_loads: Final = native_module is not None panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test") - native_size_limit: Final = 40_000_000 + native_size_limit: Final = 35_000_000 native_size_within_limit: Final = native_member.file_size <= native_size_limit validations: Final = ( (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG), @@ -223,6 +232,7 @@ def main( ("Native module loads", native_module_loads), ("Production module omits the panic test hook", panic_test_hook_absent), (f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit), + ("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies), ("Wheel contents are valid", not unexpected_members), ) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index fd7513a3937..ec7e211faa1 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -13,6 +13,7 @@ on: - ".github/workflows/codspeed.yml" - ".github/actions/setup-uv-with-retries/**" - ".github/actions/cache-cargo-build/**" + - ".github/scripts/uv_sync_with_retries.sh" pull_request: branches: - main @@ -25,6 +26,7 @@ on: - ".github/workflows/codspeed.yml" - ".github/actions/setup-uv-with-retries/**" - ".github/actions/cache-cargo-build/**" + - ".github/scripts/uv_sync_with_retries.sh" # Allow CodSpeed to trigger backtest performance analysis # in order to generate initial data workflow_dispatch: @@ -59,19 +61,27 @@ jobs: - name: Cache the Rust build uses: ./.github/actions/cache-cargo-build + with: + profile: release # Build the wheel and resolve every dependency outside the CodSpeed # runner: the same maturin build took 42 minutes inside `codspeed run` # versus under 3 minutes as a plain step (LIT-6183) - - name: Build environment + - name: Build the release wheel + run: uv build --wheel --out-dir dist + + - name: Install the wheel into the benchmark environment + run: | + UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/benchmark-venv" .github/scripts/uv_sync_with_retries.sh --frozen --no-default-groups --group benchmarks --no-install-project --python 3.12 + uv pip install --python "${RUNNER_TEMP}/benchmark-venv/bin/python" --no-deps dist/*.whl + + - name: Collect benchmarks + env: + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + LITELLM_REQUIRE_INSTALLED_WHEEL: "1" run: > - env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 - uv run --frozen --no-default-groups - --with pytest==8.3.5 - --with pytest-codspeed==4.3.0 - --with "mcp>=2.2.0,<3.0" - --with "a2a-sdk>=1.1.0,<2.0" - pytest + "${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest + --import-mode=importlib -p pytest_codspeed.plugin tests/benchmarks/ --codspeed @@ -82,13 +92,9 @@ jobs: with: mode: simulation run: > - env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 - uv run --frozen --no-default-groups - --with pytest==8.3.5 - --with pytest-codspeed==4.3.0 - --with "mcp>=2.2.0,<3.0" - --with "a2a-sdk>=1.1.0,<2.0" - pytest + env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 LITELLM_REQUIRE_INSTALLED_WHEEL=1 + "${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest + --import-mode=importlib -p pytest_codspeed.plugin tests/benchmarks/ --codspeed diff --git a/backend/Dockerfile b/backend/Dockerfile index 622fedcd70d..57e0a43a98d 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -61,6 +61,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra saml \ --python python3.13 +RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/ + RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ prisma generate --schema=./schema.prisma diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 50aa40ba220..5842cf6f1ac 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -1,8 +1,11 @@ """Guard the cost map on pull requests. -Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file, -and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named -litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models. +Every pull request whose diff against its merge base touches one of the three cost map files gets the file +checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the +map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies +and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot +(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three +files and may only add or update models. """ from __future__ import annotations @@ -108,20 +111,37 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str ) +def touches_cost_map(changed_files: Sequence[str]) -> bool: + return any(path in GUARDED_PATHS for path in changed_files) + + +def contract_for(bot: bool, changed_files: Sequence[str]) -> str: + if bot: + return "bot contract enforced" + return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched" + + def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]: + if not bot and not touches_cost_map(changed_files): + return () head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH) if isinstance(head_map, str): return (head_map,) return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ())) -def _git(*args: str) -> str: +def _git(*args: str) -> str | None: result: Final = subprocess.run(("git", *args), check=False, capture_output=True, text=True) - return result.stdout if result.returncode == 0 else "" + return result.stdout if result.returncode == 0 else None def snapshot(revision: str) -> Snapshot: - return Snapshot(*(_git("show", f"{revision}:{path}") for path in GUARDED_PATHS)) + return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS)) + + +def changed_files(base: str, head: str) -> tuple[str, ...] | None: + diff: Final = _git("diff", "--name-only", "--no-renames", base, head) + return None if diff is None else tuple(diff.splitlines()) def main(argv: Sequence[str]) -> int: @@ -131,9 +151,12 @@ def main(argv: Sequence[str]) -> int: parser.add_argument("--head-ref", required=True, help="head branch name of the pull request") args: Final = parser.parse_args(argv) bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX) - changed_files: Final = tuple(_git("diff", "--name-only", args.base, args.head).splitlines()) - failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed_files, bot) - contract: Final = "bot contract enforced" if bot else "human PR, file checks only" + changed: Final = changed_files(args.base, args.head) + if changed is None: + print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown") + return 1 + failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot) + contract: Final = contract_for(bot, changed) if failures: print(f"cost map guard failed ({contract}):") print("\n".join(f"- {failure}" for failure in failures)) diff --git a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json b/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json index d8cb122417a..af88708166f 100644 --- a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json +++ b/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json @@ -6267,6 +6267,63 @@ ], "title": "Spend update queue sizes (litellm__size)", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Requests that carried usage but were logged at $0 on a model whose pricing entry has a non-zero rate, by requested model and reason", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 430 + }, + "id": 110, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(litellm_zero_cost_requests_total[$__rate_interval])) by (requested_model, reason)", + "legendFormat": "{{requested_model}} / {{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "litellm_zero_cost_requests rate", + "type": "timeseries" } ], "preload": false, diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index f7557983b91..c4a3d3f7473 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -96,16 +96,19 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/assemblyai/", "/eu.assemblyai/", "/deepgram/", + "/fal_ai/", "/langfuse/", "/vllm/", "/mistral/", "/typesafe/", + "/openrouter/", "/nvidia_nim/", "/groq/", "/voyage/", "/cursor/", "/milvus/", "/openai_passthrough/", + "/tinyfish/", # Dynamic provider / toolset passthrough (path templates) "/{provider}/", "/toolset/", @@ -128,6 +131,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset( "/redoc", "/test", "/debug/memory/summary", + "/api/event_logging/batch", } ) diff --git a/helm/litellm/templates/ingress.yaml b/helm/litellm/templates/ingress.yaml index e6717821a57..e9f7ed4ec3f 100644 --- a/helm/litellm/templates/ingress.yaml +++ b/helm/litellm/templates/ingress.yaml @@ -61,7 +61,7 @@ "/v1/fine-tuning" "/fine-tuning" "/v1/responses" "/responses" "/v1/threads" "/threads" "/v1/assistants" "/assistants" "/v1/vector_stores" "/vector_stores" "/v1/indexes" "/v1/models" "/models" "/openai" "/engines" - "/v1/messages" "/messages" "/v1/skills" "/v1/a2a" "/a2a" + "/v1/messages" "/messages" "/v1/skills" "/v1/a2a" "/a2a" "/api/event_logging" "/v1/rerank" "/v2/rerank" "/rerank" "/v1/ocr" "/ocr" "/v1/rag" "/rag" "/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search" "/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat" diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 464edb3b104..d9425fc6bd7 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2727,9 +2727,13 @@ dependencies = [ "litellm-auth-types", "litellm-cache", "litellm-cache-response", + "litellm-cache-testing", + "reqwest 0.12.28", + "rstest", "serde_json", "tokio", "url", + "wiremock", ] [[package]] @@ -2737,6 +2741,7 @@ name = "litellm-cache-disk" version = "0.1.0" dependencies = [ "litellm-cache", + "litellm-cache-testing", "py_literal", "rand 0.8.7", "rstest", @@ -2755,8 +2760,10 @@ dependencies = [ "litellm-auth-gcp", "litellm-auth-types", "litellm-cache", + "litellm-cache-testing", "percent-encoding", "reqwest 0.12.28", + "rstest", "serde_json", "tokio", "wiremock", @@ -2767,8 +2774,8 @@ name = "litellm-cache-memory" version = "0.1.0" dependencies = [ "litellm-cache", + "litellm-cache-testing", "rstest", - "serde_json", "tokio", ] @@ -2776,9 +2783,10 @@ dependencies = [ name = "litellm-cache-qdrant-semantic" version = "0.1.0" dependencies = [ + "futures-executor", "futures-util", "litellm-cache", - "litellm-cache-response", + "litellm-cache-testing", "qdrant-client", "reqwest 0.12.28", "rstest", @@ -2797,9 +2805,11 @@ name = "litellm-cache-redis" version = "0.1.0" dependencies = [ "litellm-cache", + "litellm-cache-testing", "r2d2", "redis", "redis-test", + "rstest", "serde_json", "tokio", ] @@ -2810,10 +2820,10 @@ version = "0.1.0" dependencies = [ "litellm-cache", "litellm-cache-redis", - "litellm-cache-response", - "r2d2", + "litellm-cache-testing", "redis", "redis-test", + "rstest", "serde_json", "sha2 0.10.9", "tokio", @@ -2829,6 +2839,7 @@ dependencies = [ "py_literal", "redis", "redis-test", + "rstest", "serde", "serde_json", "sha2 0.10.9", @@ -2841,22 +2852,35 @@ version = "0.1.0" dependencies = [ "aws-credential-types", "aws-sdk-s3", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", + "futures-util", + "http 1.4.2", "litellm-auth-aws", "litellm-cache", + "litellm-cache-testing", + "reqwest 0.12.28", + "rstest", "serde_json", "tokio", "wiremock", ] +[[package]] +name = "litellm-cache-testing" +version = "0.1.0" +dependencies = [ + "litellm-cache", +] + [[package]] name = "litellm-cache-valkey-semantic" version = "0.1.0" dependencies = [ "litellm-cache", "litellm-cache-redis", - "litellm-cache-response", + "litellm-cache-testing", "redis", "redis-test", "rstest", @@ -2894,6 +2918,7 @@ dependencies = [ "litellm-host", "litellm-http", "litellm-llms", + "litellm-secrets", "litellm-types", "mime_guess", "moka", @@ -3006,6 +3031,7 @@ dependencies = [ "litellm-framing", "litellm-host", "litellm-http", + "litellm-secrets", "litellm-types", "reqwest 0.12.28", "rstest", @@ -3024,6 +3050,7 @@ dependencies = [ name = "litellm-python-bridge" version = "0.1.0" dependencies = [ + "aws-sdk-secretsmanager", "bytes", "criterion", "futures-util", @@ -3047,6 +3074,9 @@ dependencies = [ "litellm-host-python", "litellm-http", "litellm-llms", + "litellm-secrets", + "litellm-secrets-aws", + "litellm-secrets-types", "litellm-token-counter", "litellm-types", "pyo3", @@ -3059,9 +3089,26 @@ dependencies = [ "serde_json", "serde_with", "sha2 0.10.9", + "thiserror 2.0.19", "tokio", "tokio-tungstenite", "url", + "wiremock", +] + +[[package]] +name = "litellm-python-compat" +version = "0.1.0" +dependencies = [ + "criterion", + "hex", + "num-bigint 0.4.8", + "num-traits", + "rstest", + "serde", + "serde-pickle", + "serde_json", + "thiserror 2.0.19", ] [[package]] @@ -3246,6 +3293,7 @@ dependencies = [ name = "litellm-token-counter-huggingface" version = "0.1.0" dependencies = [ + "serde_json", "thiserror 2.0.19", "tokenizers", ] @@ -3254,6 +3302,9 @@ dependencies = [ name = "litellm-token-counter-tiktoken" version = "0.1.0" dependencies = [ + "base64 0.22.1", + "once_cell", + "rustc-hash", "thiserror 2.0.19", "tiktoken-rs", ] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 0e8941cb8e6..65813a35214 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -39,6 +39,7 @@ litellm-cache-disk = { path = "crates/cache-disk" } litellm-cache-redis-semantic = { path = "crates/cache-redis-semantic" } litellm-cache-response = { path = "crates/cache-response" } litellm-cache-qdrant-semantic = { path = "crates/cache-qdrant-semantic" } +litellm-cache-testing = { path = "crates/cache-testing" } litellm-token-counter = { path = "crates/token-counter" } litellm-token-counter-fast = { path = "crates/token-counter-fast" } litellm-token-counter-huggingface = { path = "crates/token-counter-huggingface" } @@ -89,7 +90,7 @@ veil = "0.3.0" [profile.release] opt-level = 3 -lto = "thin" +lto = "fat" codegen-units = 1 panic = "unwind" debug = false diff --git a/litellm-rust/crates/auth-aws/src/aws.rs b/litellm-rust/crates/auth-aws/src/aws.rs index bbcb0f016c8..cb9195ffeb6 100644 --- a/litellm-rust/crates/auth-aws/src/aws.rs +++ b/litellm-rust/crates/auth-aws/src/aws.rs @@ -621,6 +621,26 @@ mod tests { None } + #[test] + fn secret_names_cover_environment_reads() { + let seen = std::sync::Arc::new(std::sync::Mutex::new( + std::collections::BTreeSet::::new(), + )); + let recorded = seen.clone(); + let env = |name: &str| { + recorded.lock().unwrap().insert(name.to_string()); + None + }; + resolve_aws_region(None, &Map::new(), &env); + aws_auth_config(&Map::new(), &env); + assert!( + seen.lock() + .unwrap() + .iter() + .all(|name| crate::constants::SECRET_NAMES.contains(&name.as_str())) + ); + } + #[test] fn a_region_comes_from_the_call_then_the_model_then_the_environment() { let params = Map::from_iter([("aws_region_name".to_string(), Value::from("eu-west-1"))]); diff --git a/litellm-rust/crates/auth-aws/src/constants.rs b/litellm-rust/crates/auth-aws/src/constants.rs index 9e7c6bfab43..26df4f2a350 100644 --- a/litellm-rust/crates/auth-aws/src/constants.rs +++ b/litellm-rust/crates/auth-aws/src/constants.rs @@ -14,6 +14,19 @@ pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE"; pub const AWS_STS_ENDPOINT: &str = "AWS_STS_ENDPOINT"; pub const AWS_EXTERNAL_ID: &str = "AWS_EXTERNAL_ID"; pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK"; +pub const SECRET_NAMES: &[&str] = &[ + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + AWS_SESSION_TOKEN, + AWS_REGION_NAME, + AWS_REGION, + AWS_SESSION_NAME, + AWS_PROFILE_NAME, + AWS_ROLE_NAME, + AWS_WEB_IDENTITY_TOKEN, + AWS_STS_ENDPOINT, + AWS_EXTERNAL_ID, +]; /// Headers SigV4 covers, beyond the `x-amz-` / `x-amzn-` prefixes. Mirrors /// Python's `_filter_headers_for_aws_signature` allowlist. diff --git a/litellm-rust/crates/auth-azure/src/lib.rs b/litellm-rust/crates/auth-azure/src/lib.rs index 5c7c654b69d..9ed505e4160 100644 --- a/litellm-rust/crates/auth-azure/src/lib.rs +++ b/litellm-rust/crates/auth-azure/src/lib.rs @@ -3,5 +3,5 @@ mod native; mod resolve; mod types; -pub use resolve::AzureAuthService; +pub use resolve::{AzureAuthService, SECRET_NAMES}; pub use types::{AzureAuthInputs, ConfigValue}; diff --git a/litellm-rust/crates/auth-azure/src/resolve.rs b/litellm-rust/crates/auth-azure/src/resolve.rs index 4d564b6e68a..9a7afe645db 100644 --- a/litellm-rust/crates/auth-azure/src/resolve.rs +++ b/litellm-rust/crates/auth-azure/src/resolve.rs @@ -19,6 +19,17 @@ const AZURE_AUTHORITY_HOST_ENV: &str = "AZURE_AUTHORITY_HOST"; const AZURE_CREDENTIAL_ENV: &str = "AZURE_CREDENTIAL"; const AZURE_FEDERATED_TOKEN_FILE_ENV: &str = "AZURE_FEDERATED_TOKEN_FILE"; +pub const SECRET_NAMES: &[&str] = &[ + AZURE_AD_TOKEN_ENV, + AZURE_TENANT_ID_ENV, + AZURE_CLIENT_ID_ENV, + AZURE_CLIENT_SECRET_ENV, + AZURE_SCOPE_ENV, + AZURE_AUTHORITY_HOST_ENV, + AZURE_CREDENTIAL_ENV, + AZURE_FEDERATED_TOKEN_FILE_ENV, +]; + #[derive(Clone, Debug)] pub(crate) enum AzureCredentialPlan { Supplied(Sourced), @@ -440,13 +451,14 @@ fn non_empty_reference(value: &str, kind: &str) -> Result { #[cfg(test)] mod tests { + use std::collections::BTreeSet; use std::future::Future; use std::sync::{Arc, Mutex}; use serde_json::json; use super::{ - AzureAuthService, AzureCredentialPlan, AzureTokenAcquirer, oidc_reference, + AzureAuthService, AzureCredentialPlan, AzureTokenAcquirer, SECRET_NAMES, oidc_reference, resolve_reference, select_auth_plan, }; use crate::native::ValidatedAzureRequest; @@ -517,6 +529,24 @@ mod tests { assert!(matches!(plan, AzureCredentialPlan::Native(_))); } + #[test] + fn secret_names_cover_environment_reads() { + let seen = std::sync::Arc::new(std::sync::Mutex::new(BTreeSet::::new())); + let recorded = seen.clone(); + let inputs = AzureAuthInputs::default(); + select_auth_plan(&inputs, &|name| { + recorded.lock().unwrap().insert(name.to_string()); + None + }) + .unwrap(); + assert!( + seen.lock() + .unwrap() + .iter() + .all(|name| SECRET_NAMES.contains(&name.as_str())) + ); + } + #[test] fn supplied_token_does_not_require_refresh() { let params = json!({"azure_ad_token": "token"}); diff --git a/litellm-rust/crates/auth-gcp/src/lib.rs b/litellm-rust/crates/auth-gcp/src/lib.rs index 534d85acdb0..682f1af5fe1 100644 --- a/litellm-rust/crates/auth-gcp/src/lib.rs +++ b/litellm-rust/crates/auth-gcp/src/lib.rs @@ -23,6 +23,16 @@ const VERTEXAI_PROJECT_ENV: &str = "VERTEXAI_PROJECT"; const VERTEXAI_LOCATION_ENV: &str = "VERTEXAI_LOCATION"; const VERTEX_LOCATION_ENV: &str = "VERTEX_LOCATION"; +pub const SECRET_NAMES: &[&str] = &[ + VERTEX_AI_API_KEY_ENV, + VERTEXAI_API_KEY_ENV, + VERTEXAI_CREDENTIALS_ENV, + GOOGLE_APPLICATION_CREDENTIALS_ENV, + VERTEXAI_PROJECT_ENV, + VERTEXAI_LOCATION_ENV, + VERTEX_LOCATION_ENV, +]; + #[derive(Clone, Debug, Default)] pub struct VertexConfig { credentials: Option>, @@ -406,6 +416,7 @@ fn auth_acquisition_error(error: gcp_auth::Error) -> Error { #[cfg(test)] mod tests { + use std::collections::BTreeSet; use std::sync::atomic::{AtomicUsize, Ordering}; use serde_json::json; @@ -476,6 +487,27 @@ mod tests { ); } + #[tokio::test] + async fn secret_names_cover_environment_reads() { + let seen = Arc::new(std::sync::Mutex::new(BTreeSet::::new())); + let recorded = seen.clone(); + let env = |name: &str| { + recorded.lock().unwrap().insert(name.to_string()); + None + }; + let auth = auth(Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0))); + auth.validate_environment(Vec::new(), None, &VertexConfig::default(), &env) + .await + .unwrap(); + get_vertex_ai_location(&VertexConfig::default(), &env); + assert!( + seen.lock() + .unwrap() + .iter() + .all(|name| SECRET_NAMES.contains(&name.as_str())) + ); + } + #[test] fn empty_primary_values_fall_back_to_python_aliases() { let config = config(json!({ diff --git a/litellm-rust/crates/auth-types/src/secret.rs b/litellm-rust/crates/auth-types/src/secret.rs index a07fe3eaad9..7e6789deef7 100644 --- a/litellm-rust/crates/auth-types/src/secret.rs +++ b/litellm-rust/crates/auth-types/src/secret.rs @@ -1,4 +1,5 @@ use serde::Deserialize; +use std::hash::{Hash, Hasher}; use veil::Redact; #[derive(Redact, Clone, Deserialize)] @@ -23,6 +24,12 @@ impl PartialEq for SecretValue { impl Eq for SecretValue {} +impl Hash for SecretValue { + fn hash(&self, state: &mut H) { + self.0.hash(state); + } +} + #[cfg(test)] mod tests { use super::SecretValue; diff --git a/litellm-rust/crates/cache-azure-blob/Cargo.toml b/litellm-rust/crates/cache-azure-blob/Cargo.toml index 55abaff1975..baa1b0f5482 100644 --- a/litellm-rust/crates/cache-azure-blob/Cargo.toml +++ b/litellm-rust/crates/cache-azure-blob/Cargo.toml @@ -14,9 +14,14 @@ async-trait = "0.1" azure_core = "1.1.0" azure_storage_blob = "1.1.0" futures-util.workspace = true +reqwest.workspace = true tokio.workspace = true url.workspace = true [dev-dependencies] litellm-cache-response.workspace = true +litellm-cache-testing.workspace = true +rstest.workspace = true serde_json.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +wiremock = "0.6.5" diff --git a/litellm-rust/crates/cache-azure-blob/src/cache.rs b/litellm-rust/crates/cache-azure-blob/src/cache.rs index 6a872a0d6e6..489b08d485e 100644 --- a/litellm-rust/crates/cache-azure-blob/src/cache.rs +++ b/litellm-rust/crates/cache-azure-blob/src/cache.rs @@ -3,7 +3,7 @@ use std::{sync::Arc, time::Duration}; use azure_core::{ credentials::TokenCredential, error::ErrorKind, - http::{ClientOptions, RequestContent}, + http::{ClientOptions, RequestContent, Transport}, }; use azure_storage_blob::{ BlobContainerClient, BlobContainerClientOptions, @@ -11,13 +11,12 @@ use azure_storage_blob::{ }; use futures_util::{TryStreamExt, future::try_join_all}; use litellm_cache::{ - BaseCache, BatchCache, CacheCodec, CacheConnectionResult, CacheConnectionStatus, Error, - ExactCacheContext, FlushCache, + BaseCache, BatchCache, CacheCodec, DisconnectCache, Error, ExactCacheContext, FlushCache, }; use tokio::runtime::Handle; use url::Url; -use crate::credential::AzureBlobCredential; +use crate::{credential::AzureBlobCredential, transport::ReqwestTransport}; pub struct AzureBlobCache { container: BlobContainerClient, @@ -28,9 +27,11 @@ pub struct AzureBlobCache { } impl AzureBlobCache { + /// `http` is the host's pooled client; the SDK sends every request through it. pub async fn connect( account_url: &str, container: &str, + http: reqwest::Client, codec: C, runtime: Handle, ) -> Result { @@ -38,7 +39,10 @@ impl AzureBlobCache { account_url, container, Some(Arc::new(AzureBlobCredential::default())), - ClientOptions::default(), + ClientOptions { + transport: Some(Transport::new(Arc::new(ReqwestTransport(http)))), + ..ClientOptions::default() + }, codec, runtime, ) @@ -152,7 +156,11 @@ impl AzureBlobCache { } fn block_on(&self, future: impl Future) -> T { - self.runtime.block_on(future) + if Handle::try_current().is_ok() { + tokio::task::block_in_place(|| self.runtime.block_on(future)) + } else { + self.runtime.block_on(future) + } } } @@ -217,25 +225,6 @@ impl BaseCache for AzureBlobCache { .await .map(drop) } - - async fn disconnect(&self) -> Result<(), Error> { - Ok(()) - } - - async fn test_connection(&self) -> Result { - Ok(match self.container.get_properties(None).await { - Ok(_) => CacheConnectionResult { - status: CacheConnectionStatus::Success, - message: "Azure Blob cache connection test successful".into(), - error: None, - }, - Err(error) => CacheConnectionResult { - status: CacheConnectionStatus::Failed, - message: format!("Azure Blob connection failed: {error}"), - error: Some(error.to_string()), - }, - }) - } } impl BatchCache for AzureBlobCache {} @@ -250,5 +239,10 @@ impl FlushCache for AzureBlobCache { } } -#[cfg(test)] -mod tests; +impl DisconnectCache for AzureBlobCache { + /// Python closes its two SDK clients; the Rust clients hold no connection of their own + /// (the pooled transport belongs to the host), so there is nothing to release. + async fn disconnect(&self) -> Result<(), Error> { + Ok(()) + } +} diff --git a/litellm-rust/crates/cache-azure-blob/src/cache/tests.rs b/litellm-rust/crates/cache-azure-blob/src/cache/tests.rs deleted file mode 100644 index f8736ab069b..00000000000 --- a/litellm-rust/crates/cache-azure-blob/src/cache/tests.rs +++ /dev/null @@ -1,746 +0,0 @@ -use std::{ - collections::BTreeMap, - sync::{Arc, Mutex}, - time::Duration, -}; - -use azure_core::http::{ - AsyncRawResponse, Body, ClientOptions, HttpClient, Method, Request, StatusCode, Transport, - headers::{HeaderName, Headers}, -}; -use litellm_cache::{ - BaseCache, BatchCache, BatchEntry, CacheConnectionStatus, Error, ExactCacheContext, FlushCache, -}; -use litellm_cache_response::{ - CacheEntry, CacheKeyField, CacheKeyInput, ResponseCache, ResponseCacheCodec, - ResponseCacheRequest, cache_key, -}; -use serde_json::json; -use tokio::runtime::Runtime; - -use super::AzureBlobCache; - -const ACCOUNT_URL: &str = "https://example.blob.core.windows.net"; -const CONTAINER: &str = "litellm-cache"; -const IF_NONE_MATCH: HeaderName = HeaderName::from_static("if-none-match"); -const ERROR_CODE: HeaderName = HeaderName::from_static("x-ms-error-code"); - -#[derive(Clone, Debug, PartialEq, Eq)] -struct RecordedRequest { - method: Method, - path: String, - query: String, - if_none_match: Option, -} - -#[derive(Default)] -struct FakeState { - container_exists: bool, - blobs: BTreeMap>, - requests: Vec, - failing: bool, - precondition_conflicts: bool, -} - -#[derive(Clone, Default)] -struct FakeBlobService { - state: Arc>, -} - -impl std::fmt::Debug for FakeBlobService { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("FakeBlobService") - } -} - -impl FakeBlobService { - fn with_existing_container() -> Self { - let service = Self::default(); - service.state.lock().unwrap().container_exists = true; - service - } - - fn blob(&self, name: &str) -> Option> { - self.state.lock().unwrap().blobs.get(name).cloned() - } - - fn blob_names(&self) -> Vec { - self.state.lock().unwrap().blobs.keys().cloned().collect() - } - - fn seed_blob(&self, name: &str, bytes: &[u8]) { - self.state - .lock() - .unwrap() - .blobs - .insert(name.to_string(), bytes.to_vec()); - } - - fn set_failing(&self, failing: bool) { - self.state.lock().unwrap().failing = failing; - } - - fn set_precondition_conflicts(&self, enabled: bool) { - self.state.lock().unwrap().precondition_conflicts = enabled; - } - - fn requests(&self) -> Vec { - self.state.lock().unwrap().requests.clone() - } - - fn container_exists(&self) -> bool { - self.state.lock().unwrap().container_exists - } - - fn respond(status: StatusCode, error_code: Option<&str>, body: Vec) -> AsyncRawResponse { - let mut headers = Headers::new(); - if let Some(code) = error_code { - headers.insert(ERROR_CODE, code.to_string()); - } - AsyncRawResponse::from_bytes(status, headers, body) - } - - fn list_body(state: &FakeState) -> Vec { - let mut xml = String::from( - r#""#, - ); - for name in state.blobs.keys() { - xml.push_str(&format!( - "{name}BlockBlob" - )); - } - xml.push_str(""); - xml.into_bytes() - } -} - -#[async_trait::async_trait] -impl HttpClient for FakeBlobService { - async fn execute_request(&self, request: &Request) -> azure_core::Result { - let mut state = self.state.lock().unwrap(); - let path = request.url().path().to_string(); - let query = request.url().query().unwrap_or_default().to_string(); - let if_none_match = request - .headers() - .get_optional_str(&IF_NONE_MATCH) - .map(str::to_owned); - state.requests.push(RecordedRequest { - method: request.method(), - path: path.clone(), - query: query.clone(), - if_none_match: if_none_match.clone(), - }); - if state.failing { - return Ok(Self::respond( - StatusCode::Forbidden, - Some("AuthorizationFailure"), - Vec::new(), - )); - } - let container_path = format!("/{CONTAINER}"); - let blob_name = path - .strip_prefix(&format!("{container_path}/")) - .map(str::to_owned); - let is_container = path == container_path && query.contains("restype=container"); - let response = match (request.method(), is_container, blob_name) { - (Method::Put, true, None) if state.container_exists => Self::respond( - StatusCode::Conflict, - Some("ContainerAlreadyExists"), - Vec::new(), - ), - (Method::Put, true, None) => { - state.container_exists = true; - Self::respond(StatusCode::Created, None, Vec::new()) - } - (Method::Get, true, None) if query.contains("comp=list") => { - Self::respond(StatusCode::Ok, None, Self::list_body(&state)) - } - (Method::Get, true, None) if state.container_exists => { - Self::respond(StatusCode::Ok, None, Vec::new()) - } - (Method::Get, true, None) => { - Self::respond(StatusCode::NotFound, Some("ContainerNotFound"), Vec::new()) - } - (Method::Put, false, Some(name)) => { - if if_none_match.as_deref() == Some("*") && state.blobs.contains_key(&name) { - if state.precondition_conflicts { - Self::respond( - StatusCode::PreconditionFailed, - Some("ConditionNotMet"), - Vec::new(), - ) - } else { - Self::respond(StatusCode::Conflict, Some("BlobAlreadyExists"), Vec::new()) - } - } else { - let bytes = match request.body() { - Body::Bytes(bytes) => bytes.to_vec(), - Body::SeekableStream(_) => panic!("unexpected streaming upload"), - }; - state.blobs.insert(name, bytes); - Self::respond(StatusCode::Created, None, Vec::new()) - } - } - (Method::Get, false, Some(name)) => match state.blobs.get(&name) { - Some(bytes) => Self::respond(StatusCode::Ok, None, bytes.clone()), - None => Self::respond(StatusCode::NotFound, Some("BlobNotFound"), Vec::new()), - }, - (Method::Delete, false, Some(name)) => match state.blobs.remove(&name) { - Some(_) => Self::respond(StatusCode::Accepted, None, Vec::new()), - None => Self::respond(StatusCode::NotFound, Some("BlobNotFound"), Vec::new()), - }, - (method, _, _) => panic!("unexpected request {method:?} {path}?{query}"), - }; - Ok(response) - } -} - -struct Fixture { - runtime: Runtime, - service: FakeBlobService, - cache: Arc>, -} - -impl Fixture { - fn new(service: FakeBlobService) -> Self { - let runtime = Runtime::new().unwrap(); - let cache = runtime - .block_on(Self::connect(&service, runtime.handle().clone())) - .unwrap(); - Self { - runtime, - service, - cache: Arc::new(cache), - } - } - - async fn connect( - service: &FakeBlobService, - handle: tokio::runtime::Handle, - ) -> Result, Error> { - AzureBlobCache::connect_with_options( - ACCOUNT_URL, - CONTAINER, - None, - ClientOptions { - transport: Some(Transport::new(Arc::new(service.clone()))), - ..ClientOptions::default() - }, - ResponseCacheCodec, - handle, - ) - .await - } - - fn response_cache(&self) -> ResponseCache> { - ResponseCache::new(self.cache.clone()) - } - - fn stored_json(&self, key: &str) -> serde_json::Value { - serde_json::from_slice(&self.service.blob(key).expect("blob should exist")).unwrap() - } -} - -fn request(model: &str) -> ResponseCacheRequest { - ResponseCacheRequest::new(CacheKeyInput { - fields: vec![CacheKeyField { - name: "model".into(), - value: Some(model.into()), - api_parameter: true, - internal_parameter: false, - }], - preset: None, - namespace: None, - include_provider_parameters: false, - }) -} - -fn now() -> Duration { - Duration::from_secs(1_700_000_000) -} - -fn entry(value: serde_json::Value) -> CacheEntry { - CacheEntry { - timestamp: Some(1_700_000_000.5), - response: value, - } -} - -fn no_ttl() -> ExactCacheContext { - ExactCacheContext::default() -} - -fn with_ttl(seconds: u64) -> ExactCacheContext { - ExactCacheContext { - ttl: Some(Duration::from_secs(seconds)), - } -} - -#[test] -fn connect_creates_the_container_once() { - let fixture = Fixture::new(FakeBlobService::default()); - assert!(fixture.service.container_exists()); - assert_eq!( - fixture.service.requests(), - vec![RecordedRequest { - method: Method::Put, - path: format!("/{CONTAINER}"), - query: "restype=container".into(), - if_none_match: None, - }] - ); - assert_eq!(fixture.cache.account_url(), ACCOUNT_URL); - assert_eq!(fixture.cache.container_name(), CONTAINER); -} - -#[test] -fn connect_accepts_an_existing_container() { - let fixture = Fixture::new(FakeBlobService::with_existing_container()); - assert!(fixture.service.container_exists()); - assert_eq!(fixture.service.requests().len(), 1); -} - -#[test] -fn connect_accepts_account_urls_with_trailing_slash() { - let runtime = Runtime::new().unwrap(); - let service = FakeBlobService::default(); - let cache = runtime - .block_on(AzureBlobCache::connect_with_options( - "https://example.blob.core.windows.net/", - CONTAINER, - None, - ClientOptions { - transport: Some(Transport::new(Arc::new(service.clone()))), - ..ClientOptions::default() - }, - ResponseCacheCodec, - runtime.handle().clone(), - )) - .unwrap(); - assert_eq!(service.requests()[0].path, format!("/{CONTAINER}")); - assert_eq!(cache.account_url(), "https://example.blob.core.windows.net"); -} - -#[test] -fn connect_keeps_account_url_query_parameters_on_the_container_path() { - let runtime = Runtime::new().unwrap(); - let service = FakeBlobService::default(); - runtime - .block_on(AzureBlobCache::connect_with_options( - "https://example.blob.core.windows.net/?sv=2024-01-01&sig=abc", - CONTAINER, - None, - ClientOptions { - transport: Some(Transport::new(Arc::new(service.clone()))), - ..ClientOptions::default() - }, - ResponseCacheCodec, - runtime.handle().clone(), - )) - .unwrap(); - let create = &service.requests()[0]; - assert_eq!(create.path, format!("/{CONTAINER}")); - assert!(create.query.contains("sig=abc")); -} - -#[test] -fn connect_surfaces_service_failures() { - let runtime = Runtime::new().unwrap(); - let service = FakeBlobService::default(); - service.set_failing(true); - let result = runtime.block_on(Fixture::connect(&service, runtime.handle().clone())); - assert!(matches!(result, Err(Error::Unavailable))); -} - -#[test] -fn sync_set_and_get_round_trip_python_json_shape() { - let fixture = Fixture::new(FakeBlobService::default()); - let value = entry(json!({"choices": [{"message": {"content": "héllo 🌍"}}]})); - fixture - .cache - .set_cache("key-1", value.clone(), &no_ttl()) - .unwrap(); - - assert_eq!( - fixture.stored_json("key-1"), - json!({ - "timestamp": 1_700_000_000.5, - "response": {"choices": [{"message": {"content": "héllo 🌍"}}]} - }) - ); - assert_eq!( - fixture.cache.get_cache("key-1", &no_ttl()).unwrap(), - Some(value) - ); -} - -#[test] -fn sync_set_does_not_overwrite_an_existing_blob() { - let fixture = Fixture::new(FakeBlobService::default()); - fixture - .cache - .set_cache("key", entry(json!({"v": "first"})), &no_ttl()) - .unwrap(); - fixture - .cache - .set_cache("key", entry(json!({"v": "second"})), &no_ttl()) - .unwrap(); - - assert_eq!( - fixture.stored_json("key")["response"], - json!({"v": "first"}) - ); - let uploads: Vec<_> = fixture - .service - .requests() - .into_iter() - .filter(|request| request.method == Method::Put && request.path.ends_with("/key")) - .collect(); - assert_eq!(uploads.len(), 2); - assert!( - uploads - .iter() - .all(|request| request.if_none_match.as_deref() == Some("*")) - ); -} - -#[test] -fn sync_set_treats_a_precondition_conflict_as_an_existing_blob() { - let fixture = Fixture::new(FakeBlobService::default()); - fixture.service.set_precondition_conflicts(true); - fixture - .cache - .set_cache("key", entry(json!({"v": "first"})), &no_ttl()) - .unwrap(); - fixture - .cache - .set_cache("key", entry(json!({"v": "second"})), &no_ttl()) - .unwrap(); - - assert_eq!( - fixture.stored_json("key")["response"], - json!({"v": "first"}) - ); -} - -#[test] -fn async_set_overwrites_an_existing_blob() { - let fixture = Fixture::new(FakeBlobService::default()); - fixture.runtime.block_on(async { - fixture - .cache - .async_set_cache("key", entry(json!({"v": "first"})), no_ttl()) - .await - .unwrap(); - fixture - .cache - .async_set_cache("key", entry(json!({"v": "second"})), no_ttl()) - .await - .unwrap(); - assert_eq!( - fixture - .cache - .async_get_cache("key", &no_ttl()) - .await - .unwrap(), - Some(entry(json!({"v": "second"}))) - ); - }); - assert_eq!( - fixture.stored_json("key")["response"], - json!({"v": "second"}) - ); - assert!( - fixture - .service - .requests() - .iter() - .filter(|request| request.method == Method::Put && request.path.ends_with("/key")) - .all(|request| request.if_none_match.is_none()) - ); -} - -#[test] -fn missing_blobs_are_misses() { - let fixture = Fixture::new(FakeBlobService::default()); - assert_eq!(fixture.cache.get_cache("absent", &no_ttl()).unwrap(), None); - assert_eq!( - fixture - .runtime - .block_on(fixture.cache.async_get_cache("absent", &no_ttl())) - .unwrap(), - None - ); -} - -#[test] -fn ttl_is_ignored_and_entries_never_expire() { - let fixture = Fixture::new(FakeBlobService::default()); - assert_eq!(fixture.cache.get_ttl(&with_ttl(1)), None); - assert_eq!(fixture.cache.get_ttl(&no_ttl()), None); - - fixture - .cache - .set_cache("key", entry(json!("value")), &with_ttl(1)) - .unwrap(); - std::thread::sleep(Duration::from_millis(1100)); - assert_eq!( - fixture.cache.get_cache("key", &with_ttl(1)).unwrap(), - Some(entry(json!("value"))) - ); - assert!( - fixture - .service - .requests() - .iter() - .all(|request| !request.query.contains("expiry")) - ); -} - -#[test] -fn malformed_blobs_are_invalid_entries_and_response_cache_misses() { - let fixture = Fixture::new(FakeBlobService::default()); - fixture.service.seed_blob("broken-json", b"{not json"); - fixture - .service - .seed_blob("broken-utf8", &[0xff, 0xfe, 0x22]); - fixture - .service - .seed_blob("wrong-shape", br#"{"timestamp": "yesterday"}"#); - - for key in ["broken-json", "broken-utf8", "wrong-shape"] { - assert!(matches!( - fixture.cache.get_cache(key, &no_ttl()), - Err(Error::InvalidEntry) - )); - } - - let response_cache = fixture.response_cache(); - let broken = request("broken"); - fixture - .service - .seed_blob(&cache_key(&broken.key), b"{not json"); - assert_eq!(response_cache.lookup(&broken, now()).unwrap(), None); - assert_eq!( - fixture - .runtime - .block_on(response_cache.async_lookup(&broken, now())) - .unwrap(), - None - ); -} - -#[test] -fn batch_get_preserves_order_and_marks_misses_and_invalid_entries() { - let fixture = Fixture::new(FakeBlobService::default()); - fixture - .cache - .set_cache("a", entry(json!("A")), &no_ttl()) - .unwrap(); - fixture - .cache - .set_cache("c", entry(json!("C")), &no_ttl()) - .unwrap(); - fixture.service.seed_blob("bad", b"nope"); - let keys = ["c", "missing", "a", "bad"].map(String::from); - - let sync = fixture.cache.batch_get_cache(&keys, &no_ttl()).unwrap(); - assert_eq!( - sync, - vec![ - BatchEntry::Hit(entry(json!("C"))), - BatchEntry::Miss, - BatchEntry::Hit(entry(json!("A"))), - BatchEntry::Invalid, - ] - ); - - let asynchronous = fixture - .runtime - .block_on(fixture.cache.async_batch_get_cache(keys.to_vec(), no_ttl())) - .unwrap(); - assert_eq!(asynchronous, sync); - - let response_cache = fixture.response_cache(); - let requests = [request("hit"), request("missing"), request("bad")]; - response_cache - .store(&requests[0], json!("HIT"), now()) - .unwrap(); - fixture - .service - .seed_blob(&cache_key(&requests[2].key), b"nope"); - let hits = response_cache.lookup_batch(&requests, now()).unwrap(); - assert_eq!(hits.values, vec![Some(json!("HIT")), None, None]); - assert_eq!(hits.missing_indices, vec![1, 2]); - let async_hits = fixture - .runtime - .block_on(response_cache.async_lookup_batch(&requests, now())) - .unwrap(); - assert_eq!(async_hits.values, hits.values); -} - -#[test] -fn async_pipeline_writes_every_entry_with_overwrite() { - let fixture = Fixture::new(FakeBlobService::default()); - fixture.service.seed_blob("k2", b"stale"); - fixture - .runtime - .block_on(fixture.cache.async_set_cache_pipeline( - vec![ - ("k1".into(), entry(json!({"n": 1}))), - ("k2".into(), entry(json!({"n": 2}))), - ("k3".into(), entry(json!({"n": 3}))), - ], - with_ttl(30), - )) - .unwrap(); - assert_eq!(fixture.service.blob_names(), ["k1", "k2", "k3"]); - assert_eq!(fixture.stored_json("k2")["response"], json!({"n": 2})); -} - -#[test] -fn flush_deletes_every_blob_in_the_container() { - let fixture = Fixture::new(FakeBlobService::default()); - for key in ["x", "y", "z"] { - fixture - .cache - .set_cache(key, entry(json!(key)), &no_ttl()) - .unwrap(); - } - fixture.cache.flush_cache().unwrap(); - assert!(fixture.service.blob_names().is_empty()); - assert!(fixture.service.container_exists()); - - fixture - .cache - .set_cache("again", entry(json!(1)), &no_ttl()) - .unwrap(); - fixture - .runtime - .block_on(fixture.cache.async_flush_cache()) - .unwrap(); - assert!(fixture.service.blob_names().is_empty()); -} - -#[test] -fn service_failures_map_to_unavailable() { - let fixture = Fixture::new(FakeBlobService::default()); - fixture.service.set_failing(true); - assert!(matches!( - fixture.cache.get_cache("key", &no_ttl()), - Err(Error::Unavailable) - )); - assert!(matches!( - fixture.cache.set_cache("key", entry(json!(1)), &no_ttl()), - Err(Error::Unavailable) - )); - assert!(matches!( - fixture.cache.flush_cache(), - Err(Error::Unavailable) - )); - assert!(matches!( - fixture.runtime.block_on( - fixture - .cache - .async_set_cache_pipeline(vec![("k".into(), entry(json!(1)))], no_ttl()) - ), - Err(Error::Unavailable) - )); -} - -#[test] -fn test_connection_reports_container_reachability() { - let fixture = Fixture::new(FakeBlobService::default()); - let ok = fixture - .runtime - .block_on(fixture.cache.test_connection()) - .unwrap(); - assert_eq!(ok.status, CacheConnectionStatus::Success); - assert!(ok.error.is_none()); - - fixture.service.set_failing(true); - let failed = fixture - .runtime - .block_on(fixture.cache.test_connection()) - .unwrap(); - assert_eq!(failed.status, CacheConnectionStatus::Failed); - assert!(failed.error.is_some()); -} - -#[test] -fn disconnect_is_idempotent_and_keeps_data() { - let fixture = Fixture::new(FakeBlobService::default()); - fixture - .cache - .set_cache("key", entry(json!(1)), &no_ttl()) - .unwrap(); - fixture.runtime.block_on(async { - fixture.cache.disconnect().await.unwrap(); - fixture.cache.disconnect().await.unwrap(); - }); - assert_eq!( - fixture.cache.get_cache("key", &no_ttl()).unwrap(), - Some(entry(json!(1))) - ); -} - -#[test] -fn response_cache_stores_and_reads_through_the_backend() { - let fixture = Fixture::new(FakeBlobService::default()); - let response_cache = fixture.response_cache(); - let mut request = request("gpt"); - request.context = with_ttl(60); - let response = json!({"id": "chatcmpl-1"}); - response_cache - .store(&request, response.clone(), now()) - .unwrap(); - assert_eq!( - fixture.stored_json(&cache_key(&request.key)), - json!({"timestamp": 1_700_000_000.0, "response": {"id": "chatcmpl-1"}}) - ); - assert_eq!( - response_cache - .lookup(&request, now() + Duration::from_secs(3600)) - .unwrap(), - Some(response.clone()) - ); - assert_eq!( - fixture - .runtime - .block_on(response_cache.async_lookup(&request, now() + Duration::from_secs(3600))) - .unwrap(), - Some(response.clone()) - ); - fixture.runtime.block_on(async { - response_cache - .async_store(&request, json!("replaced"), now()) - .await - .unwrap(); - assert_eq!( - response_cache.async_lookup(&request, now()).await.unwrap(), - Some(json!("replaced")) - ); - response_cache.async_flush().await.unwrap(); - assert_eq!( - response_cache.async_lookup(&request, now()).await.unwrap(), - None - ); - }); -} - -#[test] -fn non_object_responses_are_written_serialized_like_python() { - let fixture = Fixture::new(FakeBlobService::default()); - fixture - .cache - .set_cache("s", entry(json!("plain")), &no_ttl()) - .unwrap(); - assert_eq!( - fixture.stored_json("s"), - json!({"timestamp": 1_700_000_000.5, "response": "\"plain\""}) - ); - assert_eq!( - fixture.cache.get_cache("s", &no_ttl()).unwrap(), - Some(entry(json!("plain"))) - ); -} diff --git a/litellm-rust/crates/cache-azure-blob/src/lib.rs b/litellm-rust/crates/cache-azure-blob/src/lib.rs index 5ae752c111d..6bcfe130576 100644 --- a/litellm-rust/crates/cache-azure-blob/src/lib.rs +++ b/litellm-rust/crates/cache-azure-blob/src/lib.rs @@ -1,5 +1,7 @@ mod cache; mod credential; +mod transport; pub use cache::AzureBlobCache; pub use credential::AzureBlobCredential; +pub use transport::ReqwestTransport; diff --git a/litellm-rust/crates/cache-azure-blob/src/transport.rs b/litellm-rust/crates/cache-azure-blob/src/transport.rs new file mode 100644 index 00000000000..ed038b8d69d --- /dev/null +++ b/litellm-rust/crates/cache-azure-blob/src/transport.rs @@ -0,0 +1,49 @@ +use azure_core::{ + error::ErrorKind, + http::{ + AsyncRawResponse, Body, HttpClient, Request, + headers::{HeaderName, HeaderValue, Headers}, + }, +}; +use futures_util::TryStreamExt; + +#[derive(Debug)] +pub struct ReqwestTransport(pub reqwest::Client); + +#[async_trait::async_trait] +impl HttpClient for ReqwestTransport { + async fn execute_request(&self, request: &Request) -> azure_core::Result { + let method = reqwest::Method::from_bytes(request.method().as_ref().as_bytes()) + .map_err(|error| azure_core::Error::new(ErrorKind::Other, error))?; + let mut outgoing = self.0.request(method, request.url().as_str()); + for (name, value) in request.headers().iter() { + outgoing = outgoing.header(name.as_str(), value.as_str()); + } + let outgoing = match request.body().clone() { + Body::Bytes(bytes) => outgoing.body(bytes), + Body::SeekableStream(stream) => outgoing.body(reqwest::Body::wrap_stream(stream)), + }; + let response = outgoing.send().await.map_err(|error| { + let kind = if error.is_connect() { + ErrorKind::Connection + } else { + ErrorKind::Io + }; + azure_core::Error::new(kind, error) + })?; + let status = response.status().as_u16().into(); + let mut headers = Headers::new(); + for (name, value) in response.headers() { + if let Ok(value) = value.to_str() { + headers.insert( + HeaderName::from(name.as_str().to_owned()), + HeaderValue::from(value.to_owned()), + ); + } + } + let body = response + .bytes_stream() + .map_err(|error| azure_core::Error::new(ErrorKind::Io, error)); + Ok(AsyncRawResponse::new(status, headers, Box::pin(body))) + } +} diff --git a/litellm-rust/crates/cache-azure-blob/tests/cache.rs b/litellm-rust/crates/cache-azure-blob/tests/cache.rs new file mode 100644 index 00000000000..b9f50b824b3 --- /dev/null +++ b/litellm-rust/crates/cache-azure-blob/tests/cache.rs @@ -0,0 +1,494 @@ +mod support; + +use std::{sync::Arc, time::Duration}; + +use azure_core::http::Method; +use litellm_cache::{ + BaseCache, BatchCache, BatchEntry, DisconnectCache, Error, ExactCacheContext, FlushCache, +}; +use litellm_cache_azure_blob::AzureBlobCache; +use litellm_cache_response::{ + CacheEntry, CacheKeyField, CacheKeyInput, ResponseCache, ResponseCacheCodec, + ResponseCacheRequest, cache_key, +}; +use rstest::{fixture, rstest}; +use serde_json::json; +use support::{ACCOUNT_URL, CONTAINER, FakeBlobService, RecordedRequest}; +use tokio::runtime::Runtime; + +type Fixture = support::Fixture; + +#[fixture] +fn fixture() -> Fixture { + Fixture::new(FakeBlobService::default(), ResponseCacheCodec) +} + +fn response_cache(fixture: &Fixture) -> ResponseCache> { + ResponseCache::new(fixture.cache.clone()) +} + +fn request(model: &str) -> ResponseCacheRequest { + ResponseCacheRequest::new(CacheKeyInput { + fields: vec![CacheKeyField { + name: "model".into(), + value: Some(model.into()), + api_parameter: true, + internal_parameter: false, + }], + preset: None, + namespace: None, + include_provider_parameters: false, + }) +} + +fn now() -> Duration { + Duration::from_secs(1_700_000_000) +} + +fn entry(value: serde_json::Value) -> CacheEntry { + CacheEntry { + timestamp: Some(1_700_000_000.5), + response: value, + } +} + +fn no_ttl() -> ExactCacheContext { + ExactCacheContext::default() +} + +fn with_ttl(seconds: u64) -> ExactCacheContext { + ExactCacheContext { + ttl: Some(Duration::from_secs(seconds)), + } +} + +fn connect_to(account_url: &str) -> (FakeBlobService, AzureBlobCache) { + let runtime = Runtime::new().unwrap(); + let service = FakeBlobService::default(); + let cache = runtime + .block_on(support::connect( + &service, + account_url, + ResponseCacheCodec, + runtime.handle().clone(), + )) + .unwrap(); + (service, cache) +} + +#[rstest] +fn connect_creates_the_container_once(fixture: Fixture) { + assert!(fixture.service.container_exists()); + assert_eq!( + fixture.service.requests(), + vec![RecordedRequest { + method: Method::Put, + path: format!("/{CONTAINER}"), + query: "restype=container".into(), + if_none_match: None, + }] + ); + assert_eq!(fixture.cache.account_url(), ACCOUNT_URL); + assert_eq!(fixture.cache.container_name(), CONTAINER); +} + +#[rstest] +fn connect_accepts_an_existing_container() { + let fixture = Fixture::new( + FakeBlobService::with_existing_container(), + ResponseCacheCodec, + ); + assert!(fixture.service.container_exists()); + assert_eq!(fixture.service.requests().len(), 1); +} + +#[rstest] +fn connect_accepts_account_urls_with_trailing_slash() { + let (service, cache) = connect_to("https://example.blob.core.windows.net/"); + assert_eq!(service.requests()[0].path, format!("/{CONTAINER}")); + assert_eq!(cache.account_url(), "https://example.blob.core.windows.net"); +} + +#[rstest] +fn connect_keeps_account_url_query_parameters_on_the_container_path() { + let (service, _) = connect_to("https://example.blob.core.windows.net/?sv=2024-01-01&sig=abc"); + let create = &service.requests()[0]; + assert_eq!(create.path, format!("/{CONTAINER}")); + assert!(create.query.contains("sig=abc")); +} + +#[rstest] +fn connect_surfaces_service_failures() { + let runtime = Runtime::new().unwrap(); + let service = FakeBlobService::default(); + service.set_failing(true); + let result = runtime.block_on(support::connect( + &service, + ACCOUNT_URL, + ResponseCacheCodec, + runtime.handle().clone(), + )); + assert!(matches!(result, Err(Error::Unavailable))); +} + +#[rstest] +fn sync_set_and_get_round_trip_python_json_shape(fixture: Fixture) { + let value = entry(json!({"choices": [{"message": {"content": "héllo 🌍"}}]})); + fixture + .cache + .set_cache("key-1", value.clone(), &no_ttl()) + .unwrap(); + + assert_eq!( + fixture.stored_json("key-1"), + json!({ + "timestamp": 1_700_000_000.5, + "response": {"choices": [{"message": {"content": "héllo 🌍"}}]} + }) + ); + assert_eq!( + fixture.cache.get_cache("key-1", &no_ttl()).unwrap(), + Some(value) + ); +} + +#[rstest] +#[case::blob_already_exists(false)] +#[case::precondition_conflict(true)] +fn sync_set_does_not_overwrite_an_existing_blob(fixture: Fixture, #[case] precondition: bool) { + fixture.service.set_precondition_conflicts(precondition); + fixture + .cache + .set_cache("key", entry(json!({"v": "first"})), &no_ttl()) + .unwrap(); + fixture + .cache + .set_cache("key", entry(json!({"v": "second"})), &no_ttl()) + .unwrap(); + + assert_eq!( + fixture.stored_json("key")["response"], + json!({"v": "first"}) + ); + let uploads: Vec<_> = fixture + .service + .requests() + .into_iter() + .filter(|request| request.method == Method::Put && request.path.ends_with("/key")) + .collect(); + assert_eq!(uploads.len(), 2); + assert!( + uploads + .iter() + .all(|request| request.if_none_match.as_deref() == Some("*")) + ); +} + +#[rstest] +fn async_set_overwrites_an_existing_blob(fixture: Fixture) { + fixture.runtime.block_on(async { + fixture + .cache + .async_set_cache("key", entry(json!({"v": "first"})), no_ttl()) + .await + .unwrap(); + fixture + .cache + .async_set_cache("key", entry(json!({"v": "second"})), no_ttl()) + .await + .unwrap(); + assert_eq!( + fixture + .cache + .async_get_cache("key", &no_ttl()) + .await + .unwrap(), + Some(entry(json!({"v": "second"}))) + ); + }); + assert_eq!( + fixture.stored_json("key")["response"], + json!({"v": "second"}) + ); + assert!( + fixture + .service + .requests() + .iter() + .filter(|request| request.method == Method::Put && request.path.ends_with("/key")) + .all(|request| request.if_none_match.is_none()) + ); +} + +#[rstest] +fn missing_blobs_are_misses(fixture: Fixture) { + assert_eq!(fixture.cache.get_cache("absent", &no_ttl()).unwrap(), None); + assert_eq!( + fixture + .runtime + .block_on(fixture.cache.async_get_cache("absent", &no_ttl())) + .unwrap(), + None + ); +} + +#[rstest] +fn ttl_is_ignored_and_entries_never_expire(fixture: Fixture) { + assert_eq!(fixture.cache.get_ttl(&with_ttl(1)), None); + assert_eq!(fixture.cache.get_ttl(&no_ttl()), None); + + fixture + .cache + .set_cache("key", entry(json!("value")), &with_ttl(1)) + .unwrap(); + std::thread::sleep(Duration::from_millis(1100)); + assert_eq!( + fixture.cache.get_cache("key", &with_ttl(1)).unwrap(), + Some(entry(json!("value"))) + ); + assert!( + fixture + .service + .requests() + .iter() + .all(|request| !request.query.contains("expiry")) + ); +} + +#[rstest] +#[case::broken_json("broken-json", b"{not json".as_slice())] +#[case::broken_utf8("broken-utf8", &[0xff, 0xfe, 0x22])] +#[case::wrong_shape("wrong-shape", br#"{"timestamp": "yesterday"}"#.as_slice())] +fn malformed_blobs_are_invalid_entries(fixture: Fixture, #[case] key: &str, #[case] bytes: &[u8]) { + fixture.service.seed_blob(key, bytes); + assert!(matches!( + fixture.cache.get_cache(key, &no_ttl()), + Err(Error::InvalidEntry) + )); +} + +#[rstest] +fn malformed_blobs_are_response_cache_misses(fixture: Fixture) { + let response_cache = response_cache(&fixture); + let broken = request("broken"); + fixture + .service + .seed_blob(&cache_key(&broken.key), b"{not json"); + assert_eq!(response_cache.lookup(&broken, now()).unwrap(), None); + assert_eq!( + fixture + .runtime + .block_on(response_cache.async_lookup(&broken, now())) + .unwrap(), + None + ); +} + +#[rstest] +fn batch_get_preserves_order_and_marks_misses_and_invalid_entries(fixture: Fixture) { + fixture + .cache + .set_cache("a", entry(json!("A")), &no_ttl()) + .unwrap(); + fixture + .cache + .set_cache("c", entry(json!("C")), &no_ttl()) + .unwrap(); + fixture.service.seed_blob("bad", b"nope"); + let keys = ["c", "missing", "a", "bad"].map(String::from); + + let sync = fixture.cache.batch_get_cache(&keys, &no_ttl()).unwrap(); + assert_eq!( + sync, + vec![ + BatchEntry::Hit(entry(json!("C"))), + BatchEntry::Miss, + BatchEntry::Hit(entry(json!("A"))), + BatchEntry::Invalid, + ] + ); + + let asynchronous = fixture + .runtime + .block_on(fixture.cache.async_batch_get_cache(keys.to_vec(), no_ttl())) + .unwrap(); + assert_eq!(asynchronous, sync); + + let response_cache = response_cache(&fixture); + let requests = [request("hit"), request("missing"), request("bad")]; + response_cache + .store(&requests[0], json!("HIT"), now()) + .unwrap(); + fixture + .service + .seed_blob(&cache_key(&requests[2].key), b"nope"); + let hits = response_cache.lookup_batch(&requests, now()).unwrap(); + assert_eq!(hits.values, vec![Some(json!("HIT")), None, None]); + assert_eq!(hits.missing_indices, vec![1, 2]); + let async_hits = fixture + .runtime + .block_on(response_cache.async_lookup_batch(&requests, now())) + .unwrap(); + assert_eq!(async_hits.values, hits.values); +} + +#[rstest] +fn async_pipeline_writes_every_entry_with_overwrite(fixture: Fixture) { + fixture.service.seed_blob("k2", b"stale"); + fixture + .runtime + .block_on(fixture.cache.async_set_cache_pipeline( + vec![ + ("k1".into(), entry(json!({"n": 1}))), + ("k2".into(), entry(json!({"n": 2}))), + ("k3".into(), entry(json!({"n": 3}))), + ], + with_ttl(30), + )) + .unwrap(); + assert_eq!(fixture.service.blob_names(), ["k1", "k2", "k3"]); + assert_eq!(fixture.stored_json("k2")["response"], json!({"n": 2})); +} + +#[rstest] +fn flush_deletes_every_blob_in_the_container(fixture: Fixture) { + for key in ["x", "y", "z"] { + fixture + .cache + .set_cache(key, entry(json!(key)), &no_ttl()) + .unwrap(); + } + fixture.cache.flush_cache().unwrap(); + assert!(fixture.service.blob_names().is_empty()); + assert!(fixture.service.container_exists()); + + fixture + .cache + .set_cache("again", entry(json!(1)), &no_ttl()) + .unwrap(); + fixture + .runtime + .block_on(fixture.cache.async_flush_cache()) + .unwrap(); + assert!(fixture.service.blob_names().is_empty()); +} + +#[rstest] +fn service_failures_map_to_unavailable(fixture: Fixture) { + fixture.service.set_failing(true); + assert!(matches!( + fixture.cache.get_cache("key", &no_ttl()), + Err(Error::Unavailable) + )); + assert!(matches!( + fixture.cache.set_cache("key", entry(json!(1)), &no_ttl()), + Err(Error::Unavailable) + )); + assert!(matches!( + fixture.cache.flush_cache(), + Err(Error::Unavailable) + )); + assert!(matches!( + fixture.runtime.block_on( + fixture + .cache + .async_set_cache_pipeline(vec![("k".into(), entry(json!(1)))], no_ttl()) + ), + Err(Error::Unavailable) + )); +} + +#[rstest] +fn disconnect_is_idempotent_and_keeps_data(fixture: Fixture) { + fixture + .cache + .set_cache("key", entry(json!(1)), &no_ttl()) + .unwrap(); + fixture.runtime.block_on(async { + fixture.cache.disconnect().await.unwrap(); + fixture.cache.disconnect().await.unwrap(); + }); + assert_eq!( + fixture.cache.get_cache("key", &no_ttl()).unwrap(), + Some(entry(json!(1))) + ); +} + +#[rstest] +fn response_cache_stores_and_reads_through_the_backend(fixture: Fixture) { + let response_cache = response_cache(&fixture); + let mut request = request("gpt"); + request.context = with_ttl(60); + let response = json!({"id": "chatcmpl-1"}); + response_cache + .store(&request, response.clone(), now()) + .unwrap(); + assert_eq!( + fixture.stored_json(&cache_key(&request.key)), + json!({"timestamp": 1_700_000_000.0, "response": {"id": "chatcmpl-1"}}) + ); + assert_eq!( + response_cache + .lookup(&request, now() + Duration::from_secs(3600)) + .unwrap(), + Some(response.clone()) + ); + assert_eq!( + fixture + .runtime + .block_on(response_cache.async_lookup(&request, now() + Duration::from_secs(3600))) + .unwrap(), + Some(response.clone()) + ); + fixture.runtime.block_on(async { + response_cache + .async_store(&request, json!("replaced"), now()) + .await + .unwrap(); + assert_eq!( + response_cache.async_lookup(&request, now()).await.unwrap(), + Some(json!("replaced")) + ); + response_cache.async_flush().await.unwrap(); + assert_eq!( + response_cache.async_lookup(&request, now()).await.unwrap(), + None + ); + }); +} + +#[rstest] +fn non_object_responses_are_written_serialized_like_python(fixture: Fixture) { + fixture + .cache + .set_cache("s", entry(json!("plain")), &no_ttl()) + .unwrap(); + assert_eq!( + fixture.stored_json("s"), + json!({"timestamp": 1_700_000_000.5, "response": "\"plain\""}) + ); + assert_eq!( + fixture.cache.get_cache("s", &no_ttl()).unwrap(), + Some(entry(json!("plain"))) + ); +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn sync_methods_block_inside_a_multi_thread_runtime() { + let service = FakeBlobService::default(); + let cache = support::connect( + &service, + ACCOUNT_URL, + ResponseCacheCodec, + tokio::runtime::Handle::current(), + ) + .await + .map(Arc::new) + .unwrap(); + cache.set_cache("key", entry(json!(1)), &no_ttl()).unwrap(); + assert_eq!( + cache.get_cache("key", &no_ttl()).unwrap(), + Some(entry(json!(1))) + ); +} diff --git a/litellm-rust/crates/cache-azure-blob/tests/contract.rs b/litellm-rust/crates/cache-azure-blob/tests/contract.rs new file mode 100644 index 00000000000..585ca26bbd7 --- /dev/null +++ b/litellm-rust/crates/cache-azure-blob/tests/contract.rs @@ -0,0 +1,81 @@ +mod support; + +use litellm_cache::{ExactCacheContext, JsonCodec}; +use litellm_cache_azure_blob::AzureBlobCache; +use litellm_cache_testing as contract; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; +use support::{ACCOUNT_URL, FakeBlobService}; +use tokio::runtime::Handle; + +#[fixture] +async fn azure() -> AzureBlobCache> { + support::connect( + &FakeBlobService::default(), + ACCOUNT_URL, + JsonCodec::new(), + Handle::current(), + ) + .await + .unwrap() +} + +#[fixture] +fn context() -> ExactCacheContext { + ExactCacheContext::default() +} + +const PREFIX: &str = "contract:"; + +// `overwrite_replaces` does not apply: sync `set_cache` never overwrites a blob, as in Python. + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn hit_and_miss( + #[future(awt)] azure: AzureBlobCache>, + context: ExactCacheContext, +) { + contract::hit_and_miss(&azure, context, PREFIX, json!({"answer": 42})).await; +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn sync_async_equivalence( + #[future(awt)] azure: AzureBlobCache>, + context: ExactCacheContext, +) { + contract::sync_async_equivalence(&azure, context, PREFIX, json!("first"), json!([2])).await; +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn pipeline_writes_every_entry( + #[future(awt)] azure: AzureBlobCache>, + context: ExactCacheContext, +) { + contract::pipeline_writes_every_entry( + &azure, + context, + PREFIX, + vec![json!("a"), json!(2), json!({"c": true})], + ) + .await; +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn batch_preserves_order( + #[future(awt)] azure: AzureBlobCache>, + context: ExactCacheContext, +) { + contract::batch_preserves_order(&azure, context, PREFIX, json!("first"), json!(2)).await; +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn flush_clears( + #[future(awt)] azure: AzureBlobCache>, + context: ExactCacheContext, +) { + contract::flush_clears(&azure, context, PREFIX, json!("value")).await; +} diff --git a/litellm-rust/crates/cache-azure-blob/tests/support/mod.rs b/litellm-rust/crates/cache-azure-blob/tests/support/mod.rs new file mode 100644 index 00000000000..f908151c22e --- /dev/null +++ b/litellm-rust/crates/cache-azure-blob/tests/support/mod.rs @@ -0,0 +1,239 @@ +#![allow(dead_code)] + +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, +}; + +use azure_core::http::{ + AsyncRawResponse, Body, ClientOptions, HttpClient, Method, Request, StatusCode, Transport, + headers::{HeaderName, Headers}, +}; +use litellm_cache::{CacheCodec, Error}; +use litellm_cache_azure_blob::AzureBlobCache; +use tokio::runtime::{Handle, Runtime}; + +pub const ACCOUNT_URL: &str = "https://example.blob.core.windows.net"; +pub const CONTAINER: &str = "litellm-cache"; +const IF_NONE_MATCH: HeaderName = HeaderName::from_static("if-none-match"); +const ERROR_CODE: HeaderName = HeaderName::from_static("x-ms-error-code"); + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecordedRequest { + pub method: Method, + pub path: String, + pub query: String, + pub if_none_match: Option, +} + +#[derive(Default)] +struct FakeState { + container_exists: bool, + blobs: BTreeMap>, + requests: Vec, + failing: bool, + precondition_conflicts: bool, +} + +#[derive(Clone, Default)] +pub struct FakeBlobService { + state: Arc>, +} + +impl std::fmt::Debug for FakeBlobService { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("FakeBlobService") + } +} + +impl FakeBlobService { + pub fn with_existing_container() -> Self { + let service = Self::default(); + service.state.lock().unwrap().container_exists = true; + service + } + + pub fn blob(&self, name: &str) -> Option> { + self.state.lock().unwrap().blobs.get(name).cloned() + } + + pub fn blob_names(&self) -> Vec { + self.state.lock().unwrap().blobs.keys().cloned().collect() + } + + pub fn seed_blob(&self, name: &str, bytes: &[u8]) { + self.state + .lock() + .unwrap() + .blobs + .insert(name.to_string(), bytes.to_vec()); + } + + pub fn set_failing(&self, failing: bool) { + self.state.lock().unwrap().failing = failing; + } + + pub fn set_precondition_conflicts(&self, enabled: bool) { + self.state.lock().unwrap().precondition_conflicts = enabled; + } + + pub fn requests(&self) -> Vec { + self.state.lock().unwrap().requests.clone() + } + + pub fn container_exists(&self) -> bool { + self.state.lock().unwrap().container_exists + } + + fn respond(status: StatusCode, error_code: Option<&str>, body: Vec) -> AsyncRawResponse { + let mut headers = Headers::new(); + if let Some(code) = error_code { + headers.insert(ERROR_CODE, code.to_string()); + } + AsyncRawResponse::from_bytes(status, headers, body) + } + + fn list_body(state: &FakeState) -> Vec { + let mut xml = String::from( + r#""#, + ); + for name in state.blobs.keys() { + xml.push_str(&format!( + "{name}BlockBlob" + )); + } + xml.push_str(""); + xml.into_bytes() + } +} + +#[async_trait::async_trait] +impl HttpClient for FakeBlobService { + async fn execute_request(&self, request: &Request) -> azure_core::Result { + let mut state = self.state.lock().unwrap(); + let path = request.url().path().to_string(); + let query = request.url().query().unwrap_or_default().to_string(); + let if_none_match = request + .headers() + .get_optional_str(&IF_NONE_MATCH) + .map(str::to_owned); + state.requests.push(RecordedRequest { + method: request.method(), + path: path.clone(), + query: query.clone(), + if_none_match: if_none_match.clone(), + }); + if state.failing { + return Ok(Self::respond( + StatusCode::Forbidden, + Some("AuthorizationFailure"), + Vec::new(), + )); + } + let container_path = format!("/{CONTAINER}"); + let blob_name = path + .strip_prefix(&format!("{container_path}/")) + .map(str::to_owned); + let is_container = path == container_path && query.contains("restype=container"); + let response = match (request.method(), is_container, blob_name) { + (Method::Put, true, None) if state.container_exists => Self::respond( + StatusCode::Conflict, + Some("ContainerAlreadyExists"), + Vec::new(), + ), + (Method::Put, true, None) => { + state.container_exists = true; + Self::respond(StatusCode::Created, None, Vec::new()) + } + (Method::Get, true, None) if query.contains("comp=list") => { + Self::respond(StatusCode::Ok, None, Self::list_body(&state)) + } + (Method::Get, true, None) if state.container_exists => { + Self::respond(StatusCode::Ok, None, Vec::new()) + } + (Method::Get, true, None) => { + Self::respond(StatusCode::NotFound, Some("ContainerNotFound"), Vec::new()) + } + (Method::Put, false, Some(name)) => { + if if_none_match.as_deref() == Some("*") && state.blobs.contains_key(&name) { + if state.precondition_conflicts { + Self::respond( + StatusCode::PreconditionFailed, + Some("ConditionNotMet"), + Vec::new(), + ) + } else { + Self::respond(StatusCode::Conflict, Some("BlobAlreadyExists"), Vec::new()) + } + } else { + let bytes = match request.body() { + Body::Bytes(bytes) => bytes.to_vec(), + Body::SeekableStream(_) => panic!("unexpected streaming upload"), + }; + state.blobs.insert(name, bytes); + Self::respond(StatusCode::Created, None, Vec::new()) + } + } + (Method::Get, false, Some(name)) => match state.blobs.get(&name) { + Some(bytes) => Self::respond(StatusCode::Ok, None, bytes.clone()), + None => Self::respond(StatusCode::NotFound, Some("BlobNotFound"), Vec::new()), + }, + (Method::Delete, false, Some(name)) => match state.blobs.remove(&name) { + Some(_) => Self::respond(StatusCode::Accepted, None, Vec::new()), + None => Self::respond(StatusCode::NotFound, Some("BlobNotFound"), Vec::new()), + }, + (method, _, _) => panic!("unexpected request {method:?} {path}?{query}"), + }; + Ok(response) + } +} + +pub async fn connect( + service: &FakeBlobService, + account_url: &str, + codec: C, + handle: Handle, +) -> Result, Error> { + AzureBlobCache::connect_with_options( + account_url, + CONTAINER, + None, + ClientOptions { + transport: Some(Transport::new(Arc::new(service.clone()))), + ..ClientOptions::default() + }, + codec, + handle, + ) + .await +} + +/// A cache on its own fake service and runtime, so sync methods run outside any runtime. +pub struct Fixture { + pub runtime: Runtime, + pub service: FakeBlobService, + pub cache: Arc>, +} + +impl Fixture { + pub fn new(service: FakeBlobService, codec: C) -> Self { + let runtime = Runtime::new().unwrap(); + let cache = runtime + .block_on(connect( + &service, + ACCOUNT_URL, + codec, + runtime.handle().clone(), + )) + .unwrap(); + Self { + runtime, + service, + cache: Arc::new(cache), + } + } + + pub fn stored_json(&self, key: &str) -> serde_json::Value { + serde_json::from_slice(&self.service.blob(key).expect("blob should exist")).unwrap() + } +} diff --git a/litellm-rust/crates/cache-azure-blob/tests/transport.rs b/litellm-rust/crates/cache-azure-blob/tests/transport.rs new file mode 100644 index 00000000000..cd1e10aa3d8 --- /dev/null +++ b/litellm-rust/crates/cache-azure-blob/tests/transport.rs @@ -0,0 +1,90 @@ +use std::sync::Arc; + +use azure_core::http::{ClientOptions, Transport}; +use litellm_cache::{BaseCache, ExactCacheContext, JsonCodec}; +use litellm_cache_azure_blob::{AzureBlobCache, ReqwestTransport}; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; +use tokio::runtime::Handle; +use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{body_json, header, method, path, query_param}, +}; + +#[fixture] +async fn server() -> MockServer { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .and(path("/litellm-cache")) + .and(query_param("restype", "container")) + .respond_with(ResponseTemplate::new(201)) + .expect(1) + .mount(&server) + .await; + server +} + +async fn connect(server: &MockServer) -> AzureBlobCache> { + AzureBlobCache::connect_with_options( + &server.uri(), + "litellm-cache", + None, + ClientOptions { + transport: Some(Transport::new(Arc::new(ReqwestTransport( + reqwest::Client::new(), + )))), + ..ClientOptions::default() + }, + JsonCodec::new(), + Handle::current(), + ) + .await + .unwrap() +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn uploads_go_through_the_host_client(#[future(awt)] server: MockServer) { + Mock::given(method("PUT")) + .and(path("/litellm-cache/key")) + .and(header("if-none-match", "*")) + .and(body_json(json!({"answer": 1}))) + .respond_with(ResponseTemplate::new(201)) + .expect(1) + .mount(&server) + .await; + connect(&server) + .await + .set_cache("key", json!({"answer": 1}), &ExactCacheContext::default()) + .unwrap(); +} + +#[rstest] +#[case::hit( + ResponseTemplate::new(200).set_body_json(json!({"answer": 2})), + Some(json!({"answer": 2})) +)] +#[case::blob_not_found( + ResponseTemplate::new(404).insert_header("x-ms-error-code", "BlobNotFound"), + None +)] +#[tokio::test(flavor = "multi_thread")] +async fn downloads_map_the_host_client_response( + #[future(awt)] server: MockServer, + #[case] response: ResponseTemplate, + #[case] expected: Option, +) { + Mock::given(method("GET")) + .and(path("/litellm-cache/key")) + .respond_with(response) + .mount(&server) + .await; + assert_eq!( + connect(&server) + .await + .async_get_cache("key", &ExactCacheContext::default()) + .await + .unwrap(), + expected + ); +} diff --git a/litellm-rust/crates/cache-disk/Cargo.toml b/litellm-rust/crates/cache-disk/Cargo.toml index b96994b3b55..5cb75f7f129 100644 --- a/litellm-rust/crates/cache-disk/Cargo.toml +++ b/litellm-rust/crates/cache-disk/Cargo.toml @@ -15,5 +15,6 @@ serde_json.workspace = true tokio.workspace = true [dev-dependencies] +litellm-cache-testing.workspace = true rstest.workspace = true tempfile = "3.27.0" diff --git a/litellm-rust/crates/cache-disk/src/cache.rs b/litellm-rust/crates/cache-disk/src/cache.rs index 8e1223309b4..b8a2c6ed1e4 100644 --- a/litellm-rust/crates/cache-disk/src/cache.rs +++ b/litellm-rust/crates/cache-disk/src/cache.rs @@ -5,8 +5,8 @@ use std::{ }; use litellm_cache::{ - BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, CacheConnectionStatus, - CounterCache, DeleteCache, Error, ExactCacheContext, FlushCache, + BaseCache, BatchCache, BatchEntry, CacheCodec, CounterCache, DeleteCache, DisconnectCache, + Error, ExactCacheContext, FlushCache, }; use crate::{DiskStore, DiskcacheSqliteStore, PythonDiskCacheAdapter, StoredValue, ValueAdapter}; @@ -150,29 +150,6 @@ impl BaseCache for DiskCache Result<(), Error> { - Ok(()) - } - - async fn test_connection(&self) -> Result { - let result = Self::run_blocking(Arc::clone(&self.store), |store| { - store.probe().map(|_| CacheConnectionResult { - status: CacheConnectionStatus::Success, - message: "Disk cache connection test successful".into(), - error: None, - }) - }) - .await; - Ok(match result { - Ok(result) => result, - Err(error) => CacheConnectionResult { - status: CacheConnectionStatus::Failed, - message: format!("Disk cache connection failed: {error}"), - error: Some(error.to_string()), - }, - }) - } } impl BatchCache for DiskCache { @@ -241,9 +218,13 @@ impl FlushCache for DiskCache, D: DiskStore, A: ValueAdapter> CounterCache - for DiskCache -{ +impl DisconnectCache for DiskCache { + async fn disconnect(&self) -> Result<(), Error> { + Ok(()) + } +} + +impl CounterCache for DiskCache { fn increment_cache( &self, key: &str, @@ -264,6 +245,7 @@ impl, D: DiskStore, A: ValueAdapter> CounterCache key: &str, amount: f64, context: ExactCacheContext, + _refresh_ttl: bool, ) -> Result { let key = key.to_string(); let adapter = Arc::clone(&self.adapter); diff --git a/litellm-rust/crates/cache-disk/src/sqlite.rs b/litellm-rust/crates/cache-disk/src/sqlite.rs index 9a36f8af6ad..24c2be2e1c5 100644 --- a/litellm-rust/crates/cache-disk/src/sqlite.rs +++ b/litellm-rust/crates/cache-disk/src/sqlite.rs @@ -544,18 +544,6 @@ impl DiskStore for DiskcacheSqliteStore { } } } - - fn probe(&self) -> Result<(), Error> { - let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; - connection - .query_row( - "SELECT value FROM Settings WHERE key = 'count'", - [], - |row| row.get::<_, i64>(0), - ) - .map(|_| ()) - .map_err(|_| Error::Unavailable) - } } fn default_settings() -> HashMap { diff --git a/litellm-rust/crates/cache-disk/src/store.rs b/litellm-rust/crates/cache-disk/src/store.rs index ed167317cf0..b5c12003cee 100644 --- a/litellm-rust/crates/cache-disk/src/store.rs +++ b/litellm-rust/crates/cache-disk/src/store.rs @@ -29,5 +29,4 @@ pub trait DiskStore: Send + Sync + 'static { now: f64, apply: &mut dyn FnMut(Option) -> Result<(StoredValue, Option), Error>, ) -> Result<(), Error>; - fn probe(&self) -> Result<(), Error>; } diff --git a/litellm-rust/crates/cache-disk/tests/cache.rs b/litellm-rust/crates/cache-disk/tests/cache.rs index dd1f2b1f04e..8c817a8a829 100644 --- a/litellm-rust/crates/cache-disk/tests/cache.rs +++ b/litellm-rust/crates/cache-disk/tests/cache.rs @@ -7,8 +7,8 @@ use std::{ }; use litellm_cache::{ - BaseCache, BatchCache, BatchEntry, CacheCodec, CounterCache, DeleteCache, ExactCacheContext, - FlushCache, JsonCodec, + BaseCache, BatchCache, BatchEntry, CacheCodec, CounterCache, DeleteCache, DisconnectCache, + ExactCacheContext, FlushCache, JsonCodec, }; use litellm_cache_disk::{DiskCache, DiskStore, DiskcacheSqliteStore, StoredValue, ValueAdapter}; use rstest::{fixture, rstest}; @@ -395,7 +395,7 @@ fn delete_flush_and_spilled_file_replacement_clean_up_storage(sandbox: Sandbox) #[rstest] #[tokio::test] -async fn async_operations_connection_and_delete_match_sync_operations(sandbox: Sandbox) { +async fn async_operations_disconnect_and_delete_match_sync_operations(sandbox: Sandbox) { let cache = sandbox.cache::(); let context = ExactCacheContext { ttl: Some(Duration::from_secs(60)), @@ -424,8 +424,101 @@ async fn async_operations_connection_and_delete_match_sync_operations(sandbox: S ); cache.async_delete_cache("a").await.unwrap(); cache.async_flush_cache().await.unwrap(); + cache.disconnect().await.unwrap(); +} + +#[derive(Clone, Copy, Debug)] +enum Increment { + Sync, + Async { refresh_ttl: bool }, +} + +impl Increment { + async fn apply( + self, + cache: &DiskCache>, + key: &str, + amount: f64, + context: ExactCacheContext, + ) -> f64 { + match self { + Self::Sync => cache.increment_cache(key, amount, context).unwrap(), + Self::Async { refresh_ttl } => cache + .async_increment(key, amount, context, refresh_ttl) + .await + .unwrap(), + } + } +} + +#[rstest] +#[case::sync_missing(Increment::Sync, None, 3.0, 3.0)] +#[case::sync_existing_int(Increment::Sync, Some(json!(7)), 5.0, 12.0)] +#[case::sync_non_int(Increment::Sync, Some(json!("not-a-number")), 4.0, 4.0)] +#[case::async_missing(Increment::Async { refresh_ttl: false }, None, 2.0, 2.0)] +#[case::async_existing_int(Increment::Async { refresh_ttl: false }, Some(json!(10)), 5.0, 15.0)] +#[case::async_non_int(Increment::Async { refresh_ttl: false }, Some(json!("corrupt")), 9.0, 9.0)] +#[case::async_refresh_ttl_is_ignored(Increment::Async { refresh_ttl: true }, Some(json!(1)), 1.0, 2.0)] +#[tokio::test] +async fn increments_read_back_through_get_cache( + sandbox: Sandbox, + #[case] increment: Increment, + #[case] initial: Option, + #[case] amount: f64, + #[case] expected: f64, +) { + let cache = sandbox.cache::(); + let context = ExactCacheContext::default(); + if let Some(initial) = initial { + cache + .async_set_cache("counter", initial, context.clone()) + .await + .unwrap(); + } assert_eq!( - cache.test_connection().await.unwrap().status, - litellm_cache::CacheConnectionStatus::Success + increment + .apply(&cache, "counter", amount, context.clone()) + .await, + expected + ); + assert_eq!( + cache.get_cache("counter", &context).unwrap(), + Some(json!(expected as i64)) ); } + +#[rstest] +#[case::without_refresh(false)] +#[case::with_refresh(true)] +#[tokio::test] +async fn async_increment_rewrites_ttl_on_every_write(sandbox: Sandbox, #[case] refresh_ttl: bool) { + let cache = sandbox.cache::(); + let expiry = || { + sandbox + .db() + .query_row( + "SELECT expire_time IS NOT NULL FROM Cache WHERE key = 'counter'", + [], + |row| row.get::<_, bool>(0), + ) + .unwrap() + }; + let ttl = ExactCacheContext { + ttl: Some(Duration::from_secs(60)), + }; + cache + .async_increment("counter", 1.0, ttl.clone(), refresh_ttl) + .await + .unwrap(); + assert!(expiry()); + cache + .async_increment("counter", 1.0, ExactCacheContext::default(), refresh_ttl) + .await + .unwrap(); + assert!(!expiry()); + cache + .async_increment("counter", 1.0, ttl, refresh_ttl) + .await + .unwrap(); + assert!(expiry()); +} diff --git a/litellm-rust/crates/cache-disk/tests/contract.rs b/litellm-rust/crates/cache-disk/tests/contract.rs new file mode 100644 index 00000000000..257252cb90c --- /dev/null +++ b/litellm-rust/crates/cache-disk/tests/contract.rs @@ -0,0 +1,82 @@ +use litellm_cache::{ExactCacheContext, JsonCodec}; +use litellm_cache_disk::DiskCache; +use litellm_cache_testing as contract; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; +use tempfile::TempDir; + +struct Disk { + cache: DiskCache>, + _directory: TempDir, +} + +#[fixture] +fn disk() -> Disk { + let directory = tempfile::tempdir().unwrap(); + Disk { + cache: DiskCache::open(directory.path(), JsonCodec::new()).unwrap(), + _directory: directory, + } +} + +#[fixture] +fn context() -> ExactCacheContext { + ExactCacheContext::default() +} + +const PREFIX: &str = "contract:"; + +#[rstest] +#[tokio::test] +async fn hit_and_miss(disk: Disk, context: ExactCacheContext) { + contract::hit_and_miss(&disk.cache, context, PREFIX, json!({"answer": 42})).await; +} + +#[rstest] +#[tokio::test] +async fn sync_async_equivalence(disk: Disk, context: ExactCacheContext) { + contract::sync_async_equivalence(&disk.cache, context, PREFIX, json!("first"), json!([2])) + .await; +} + +#[rstest] +#[tokio::test] +async fn overwrite_replaces(disk: Disk, context: ExactCacheContext) { + contract::overwrite_replaces(&disk.cache, context, PREFIX, json!(1), json!({"b": 2})).await; +} + +#[rstest] +#[tokio::test] +async fn pipeline_writes_every_entry(disk: Disk, context: ExactCacheContext) { + contract::pipeline_writes_every_entry( + &disk.cache, + context, + PREFIX, + vec![json!("a"), json!(2), json!({"c": true})], + ) + .await; +} + +#[rstest] +#[tokio::test] +async fn batch_preserves_order(disk: Disk, context: ExactCacheContext) { + contract::batch_preserves_order(&disk.cache, context, PREFIX, json!("first"), json!(2)).await; +} + +#[rstest] +#[tokio::test] +async fn delete_removes_key(disk: Disk, context: ExactCacheContext) { + contract::delete_removes_key(&disk.cache, context, PREFIX, json!("value")).await; +} + +#[rstest] +#[tokio::test] +async fn flush_clears(disk: Disk, context: ExactCacheContext) { + contract::flush_clears(&disk.cache, context, PREFIX, json!("value")).await; +} + +#[rstest] +#[tokio::test] +async fn counter_accumulates(disk: Disk, context: ExactCacheContext) { + contract::counter_accumulates(&disk.cache, context, PREFIX).await; +} diff --git a/litellm-rust/crates/cache-gcs/Cargo.toml b/litellm-rust/crates/cache-gcs/Cargo.toml index 4ec60bcfa3b..da0acf554f9 100644 --- a/litellm-rust/crates/cache-gcs/Cargo.toml +++ b/litellm-rust/crates/cache-gcs/Cargo.toml @@ -15,6 +15,8 @@ reqwest.workspace = true tokio.workspace = true [dev-dependencies] +litellm-cache-testing.workspace = true +rstest.workspace = true serde_json.workspace = true tokio.workspace = true wiremock = "0.6.5" diff --git a/litellm-rust/crates/cache-gcs/src/cache.rs b/litellm-rust/crates/cache-gcs/src/cache.rs index 65282ac99d5..a8a7fbc9a7b 100644 --- a/litellm-rust/crates/cache-gcs/src/cache.rs +++ b/litellm-rust/crates/cache-gcs/src/cache.rs @@ -2,7 +2,7 @@ use std::{future::Future, sync::Arc, time::Duration}; use futures_util::future::try_join_all; use litellm_cache::{ - BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, Error, ExactCacheContext, + BaseCache, BatchCache, BatchEntry, CacheCodec, DisconnectCache, Error, ExactCacheContext, FlushCache, }; use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode}; @@ -53,25 +53,25 @@ pub struct GcsCache { } impl GcsCache { - pub fn new(config: GcsConfig, codec: S) -> Result { + pub fn new(config: GcsConfig, client: Client, codec: S) -> Self { let token = Arc::new(GcpTokenSource::new(config.path_service_account.clone())); - Self::with_token_source(config, codec, token) + Self::with_token_source(config, client, codec, token) } pub fn with_token_source( config: GcsConfig, + client: Client, codec: S, token: Arc, - ) -> Result { - let client = Client::builder().build().map_err(|_| Error::Unavailable)?; + ) -> Self { let key_prefix = key_prefix(config.gcs_path.as_deref()); - Ok(Self { + Self { config, key_prefix, client, token, codec, - }) + } } pub fn bucket_name(&self) -> &str { @@ -154,26 +154,26 @@ impl GcsCache { F: Future> + Send, T: Send, { - let run = || { + let run = |future: F| { tokio::runtime::Builder::new_current_thread() .enable_all() .build() .map_err(|_| Error::Unavailable) .and_then(|runtime| runtime.block_on(future)) }; - if let Ok(handle) = tokio::runtime::Handle::try_current() { - if handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread { - return tokio::task::block_in_place(run); + match tokio::runtime::Handle::try_current() { + Ok(handle) if handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread => { + tokio::task::block_in_place(|| handle.block_on(future)) } - return std::thread::scope(|scope| { + Ok(_) => std::thread::scope(|scope| { scope - .spawn(run) + .spawn(|| run(future)) .join() .map_err(|_| Error::Unavailable) .and_then(|result| result) - }); + }), + Err(_) => run(future), } - run() } } @@ -222,14 +222,12 @@ impl BaseCache for GcsCache { .await .map(|_| ()) } +} +impl DisconnectCache for GcsCache { async fn disconnect(&self) -> Result<(), Error> { Ok(()) } - - async fn test_connection(&self) -> Result { - Err(Error::UnsupportedOperation) - } } impl BatchCache for GcsCache { diff --git a/litellm-rust/crates/cache-gcs/tests/cache.rs b/litellm-rust/crates/cache-gcs/tests/cache.rs index 45eecf01cec..cdce6a00bdd 100644 --- a/litellm-rust/crates/cache-gcs/tests/cache.rs +++ b/litellm-rust/crates/cache-gcs/tests/cache.rs @@ -1,37 +1,32 @@ -use std::{sync::Arc, time::Duration}; +mod support; + +use std::{future::Future, pin::Pin, sync::Arc, time::Duration}; use litellm_cache::{ - BaseCache, BatchCache, BatchEntry, CacheContext, Error, ExactCacheContext, FlushCache, - JsonCodec, + BaseCache, BatchCache, BatchEntry, CacheContext, DisconnectCache, Error, ExactCacheContext, + FlushCache, }; -use litellm_cache_gcs::{GcsCache, GcsConfig, StaticTokenSource, TokenSource, key_prefix}; -use serde_json::json; +use litellm_cache_gcs::{GcsCache, GcsConfig, TokenSource, key_prefix}; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; +use support::FakeBucket; use wiremock::{ Mock, MockServer, ResponseTemplate, matchers::{body_bytes, header, method, path, query_param}, }; -fn config(server: &MockServer, gcs_path: Option<&str>) -> GcsConfig { - GcsConfig { - bucket_name: "bucket".into(), - gcs_path: gcs_path.map(str::to_string), - path_service_account: None, - endpoint: server.uri(), - } +#[fixture] +async fn server() -> MockServer { + MockServer::start().await } -fn cache(server: &MockServer, gcs_path: Option<&str>) -> GcsCache> { - GcsCache::with_token_source( - config(server, gcs_path), - JsonCodec::new(), - Arc::new(StaticTokenSource("tok".into())), - ) - .unwrap() +fn context() -> ExactCacheContext { + ExactCacheContext::default() } +#[rstest] #[tokio::test] -async fn set_writes_encoded_object_and_headers() { - let server = MockServer::start().await; +async fn set_writes_encoded_object_and_headers(#[future(awt)] server: MockServer) { Mock::given(method("POST")) .and(path("/upload/storage/v1/b/bucket/o")) .and(query_param("uploadType", "media")) @@ -42,12 +37,8 @@ async fn set_writes_encoded_object_and_headers() { .expect(1) .mount(&server) .await; - cache(&server, Some("cache/")) - .set_cache( - "team:a b/c", - json!({"value": "entry"}), - &ExactCacheContext::default(), - ) + support::cache(&server, Some("cache/")) + .set_cache("team:a b/c", json!({"value": "entry"}), &context()) .unwrap(); let requests = server.received_requests().await.unwrap(); assert_eq!(requests.len(), 1); @@ -57,103 +48,108 @@ async fn set_writes_encoded_object_and_headers() { ); } +#[rstest] +#[case::hit( + "hit", + ResponseTemplate::new(200).set_body_json(json!({"value": "entry"})), + Ok(Some(json!({"value": "entry"}))) +)] +#[case::missing("missing", ResponseTemplate::new(404), Ok(None))] +#[case::server_error("server-error", ResponseTemplate::new(500), Err(Error::Unavailable))] +#[case::invalid( + "invalid", + ResponseTemplate::new(200).set_body_string("not json"), + Err(Error::InvalidEntry) +)] #[tokio::test] -async fn get_maps_statuses_and_decode_failures() { - let server = MockServer::start().await; +async fn get_maps_statuses_and_decode_failures( + #[future(awt)] server: MockServer, + #[case] key: &str, + #[case] response: ResponseTemplate, + #[case] expected: Result, Error>, +) { Mock::given(method("GET")) - .and(path("/storage/v1/b/bucket/o/hit")) + .and(path(format!("/storage/v1/b/bucket/o/{key}"))) .and(query_param("alt", "media")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"}))) + .respond_with(response) .mount(&server) .await; - Mock::given(method("GET")) - .and(path("/storage/v1/b/bucket/o/missing")) - .respond_with(ResponseTemplate::new(404)) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path("/storage/v1/b/bucket/o/server-error")) - .respond_with(ResponseTemplate::new(500)) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path("/storage/v1/b/bucket/o/invalid")) - .respond_with(ResponseTemplate::new(200).set_body_string("not json")) - .mount(&server) - .await; - - let cache = cache(&server, None); - assert_eq!( - cache - .get_cache("hit", &ExactCacheContext::default()) - .unwrap(), - Some(json!({"value": "entry"})) - ); - assert_eq!( - cache - .get_cache("missing", &ExactCacheContext::default()) - .unwrap(), - None - ); - assert_eq!( - cache - .get_cache("server-error", &ExactCacheContext::default()) - .unwrap_err(), - Error::Unavailable - ); - assert_eq!( - cache - .get_cache("invalid", &ExactCacheContext::default()) - .unwrap_err(), - Error::InvalidEntry - ); + let cache = support::cache(&server, None); + assert_eq!(cache.get_cache(key, &context()), expected); + assert_eq!(cache.async_get_cache(key, &context()).await, expected); } -#[test] -fn key_prefix_normalizes_paths() { - assert_eq!(key_prefix(None), ""); - assert_eq!(key_prefix(Some("a/b/")), "a/b/"); - assert_eq!(key_prefix(Some("a/b")), "a/b/"); - assert_eq!(key_prefix(Some("")), ""); +#[rstest] +#[case::none(None, "")] +#[case::trailing_slash(Some("a/b/"), "a/b/")] +#[case::no_trailing_slash(Some("a/b"), "a/b/")] +#[case::empty(Some(""), "")] +fn key_prefix_normalizes_paths(#[case] gcs_path: Option<&str>, #[case] expected: &str) { + assert_eq!(key_prefix(gcs_path), expected); } +#[rstest] #[tokio::test] -async fn object_names_use_python_quote_encoding() { - let server = MockServer::start().await; +async fn cache_exposes_its_configuration(#[future(awt)] server: MockServer) { + let cache = GcsCache::new( + GcsConfig { + path_service_account: Some("/secrets/sa.json".into()), + ..support::config(&server, Some("folder")) + }, + reqwest::Client::new(), + litellm_cache::JsonCodec::::new(), + ); + assert_eq!(cache.bucket_name(), "bucket"); + assert_eq!(cache.key_prefix(), "folder/"); + assert_eq!(cache.path_service_account(), Some("/secrets/sa.json")); + assert_eq!(cache.object_name("k"), "folder/k"); +} + +#[rstest] +#[case::punctuation("a~b-c_d.e/f g%h", "uploadType=media&name=p%2Fa~b-c_d.e%2Ff%20g%25h")] +#[case::utf8("ключ", "uploadType=media&name=p%2F%D0%BA%D0%BB%D1%8E%D1%87")] +#[tokio::test] +async fn object_names_use_python_quote_encoding( + #[future(awt)] server: MockServer, + #[case] key: &str, + #[case] query: &str, +) { Mock::given(method("POST")) .and(path("/upload/storage/v1/b/bucket/o")) .and(query_param("uploadType", "media")) .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + support::cache(&server, Some("p/")) + .async_set_cache(key, json!({"value": key}), context()) + .await + .unwrap(); + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests[0].url.query(), Some(query)); +} + +#[rstest] +#[tokio::test] +async fn object_names_are_encoded_in_the_download_path(#[future(awt)] server: MockServer) { + Mock::given(method("GET")) + .and(path("/storage/v1/b/bucket/o/p%2Fa%3Ab%20c")) + .and(query_param("alt", "media")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!(1))) .expect(2) .mount(&server) .await; - let cache = cache(&server, Some("p/")); - cache - .set_cache( - "a~b-c_d.e/f g%h", - json!({"value": "punctuation"}), - &ExactCacheContext::default(), - ) - .unwrap(); - cache - .set_cache( - "ключ", - json!({"value": "utf8"}), - &ExactCacheContext::default(), - ) - .unwrap(); - let requests = server.received_requests().await.unwrap(); - let queries: Vec<_> = requests - .iter() - .filter_map(|request| request.url.query()) - .collect(); - assert!(queries.contains(&"uploadType=media&name=p%2Fa~b-c_d.e%2Ff%20g%25h")); - assert!(queries.contains(&"uploadType=media&name=p%2F%D0%BA%D0%BB%D1%8E%D1%87")); + let cache = support::cache(&server, Some("p")); + assert_eq!(cache.get_cache("a:b c", &context()), Ok(Some(json!(1)))); + assert_eq!( + cache.async_get_cache("a:b c", &context()).await, + Ok(Some(json!(1))) + ); } +#[rstest] #[tokio::test] -async fn ignores_ttl_and_writes_pipeline_concurrently() { - let server = MockServer::start().await; +async fn ignores_ttl_and_writes_pipeline_concurrently(#[future(awt)] server: MockServer) { for key in ["one", "two", "three"] { Mock::given(method("POST")) .and(path("/upload/storage/v1/b/bucket/o")) @@ -164,12 +160,10 @@ async fn ignores_ttl_and_writes_pipeline_concurrently() { .mount(&server) .await; } - let cache = cache(&server, None); - assert_eq!(cache.get_ttl(&ExactCacheContext::default()), None); - assert_eq!( - cache.get_ttl(&ExactCacheContext::default().with_ttl(Some(Duration::from_secs(5)))), - None - ); + let cache = support::cache(&server, None); + let with_ttl = context().with_ttl(Some(Duration::from_secs(5))); + assert_eq!(cache.get_ttl(&context()), None); + assert_eq!(cache.get_ttl(&with_ttl), None); cache .async_set_cache_pipeline( vec![ @@ -177,15 +171,15 @@ async fn ignores_ttl_and_writes_pipeline_concurrently() { ("two".into(), json!({"key": "two"})), ("three".into(), json!({"key": "three"})), ], - ExactCacheContext::default().with_ttl(Some(Duration::from_secs(5))), + with_ttl, ) .await .unwrap(); } +#[rstest] #[tokio::test] -async fn async_batch_get_preserves_hits_misses_and_invalid_entries() { - let server = MockServer::start().await; +async fn batch_get_preserves_hits_misses_and_invalid_entries(#[future(awt)] server: MockServer) { Mock::given(method("GET")) .and(path("/storage/v1/b/bucket/o/hit")) .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"}))) @@ -201,122 +195,83 @@ async fn async_batch_get_preserves_hits_misses_and_invalid_entries() { .respond_with(ResponseTemplate::new(200).set_body_string("not json")) .mount(&server) .await; + let cache = support::cache(&server, None); + let keys = vec!["hit".to_string(), "missing".into(), "invalid".into()]; + let expected = vec![ + BatchEntry::Hit(json!({"value": "entry"})), + BatchEntry::Miss, + BatchEntry::Invalid, + ]; + assert_eq!(cache.batch_get_cache(&keys, &context()).unwrap(), expected); assert_eq!( - cache(&server, None) - .async_batch_get_cache( - vec!["hit".into(), "missing".into(), "invalid".into()], - ExactCacheContext::default(), - ) - .await - .unwrap(), - vec![ - BatchEntry::Hit(json!({"value": "entry"})), - BatchEntry::Miss, - BatchEntry::Invalid, - ] + cache.async_batch_get_cache(keys, context()).await.unwrap(), + expected ); } +#[rstest] #[tokio::test] -async fn lifecycle_operations_are_noops_and_connection_test_is_unsupported() { - let server = MockServer::start().await; - let cache = cache(&server, None); +async fn flush_and_disconnect_are_noops_like_python(#[future(awt)] server: MockServer) { + let cache = support::cache(&server, None); assert_eq!(cache.flush_cache(), Ok(())); + assert_eq!(cache.async_flush_cache().await, Ok(())); assert_eq!(cache.disconnect().await, Ok(())); - assert_eq!( - cache.test_connection().await, - Err(Error::UnsupportedOperation) - ); + assert!(server.received_requests().await.unwrap().is_empty()); } -#[test] +fn round_trip(cache: &support::JsonGcsCache) -> Result, Error> { + cache.set_cache("key", json!({"value": "entry"}), &context())?; + cache.get_cache("key", &context()) +} + +#[rstest] fn sync_operations_work_without_an_active_runtime() { let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() .unwrap(); - let server = runtime.block_on(MockServer::start()); - runtime.block_on( - Mock::given(method("POST")) - .and(path("/upload/storage/v1/b/bucket/o")) - .respond_with(ResponseTemplate::new(200)) - .mount(&server), - ); - runtime.block_on( - Mock::given(method("GET")) - .and(path("/storage/v1/b/bucket/o/key")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"}))) - .mount(&server), - ); - let cache = cache(&server, None); - cache - .set_cache( - "key", - json!({"value": "entry"}), - &ExactCacheContext::default(), - ) - .unwrap(); - assert_eq!( - cache - .get_cache("key", &ExactCacheContext::default()) - .unwrap(), - Some(json!({"value": "entry"})) - ); + let server = runtime.block_on(FakeBucket::serve()); + let cache = support::cache(&server, None); + assert_eq!(round_trip(&cache), Ok(Some(json!({"value": "entry"})))); } +#[rstest] #[tokio::test(flavor = "multi_thread")] async fn sync_operations_work_inside_a_multi_thread_runtime() { - let server = MockServer::start().await; - Mock::given(method("POST")) - .and(path("/upload/storage/v1/b/bucket/o")) - .respond_with(ResponseTemplate::new(200)) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path("/storage/v1/b/bucket/o/key")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({"value": "entry"}))) - .mount(&server) - .await; - let cache = cache(&server, None); - cache - .set_cache( - "key", - json!({"value": "entry"}), - &ExactCacheContext::default(), - ) - .unwrap(); - assert_eq!( - cache - .get_cache("key", &ExactCacheContext::default()) - .unwrap(), - Some(json!({"value": "entry"})) - ); + let server = FakeBucket::serve().await; + let cache = support::cache(&server, None); + assert_eq!(round_trip(&cache), Ok(Some(json!({"value": "entry"})))); +} + +#[rstest] +#[tokio::test] +async fn sync_operations_work_inside_a_current_thread_runtime() { + let server = FakeBucket::serve().await; + let cache = support::cache(&server, None); + assert_eq!(round_trip(&cache), Ok(Some(json!({"value": "entry"})))); } struct FailingTokenSource; impl TokenSource for FailingTokenSource { - fn bearer_token( - &self, - ) -> std::pin::Pin> + Send + '_>> - { + fn bearer_token(&self) -> Pin> + Send + '_>> { Box::pin(async { Err(Error::Unavailable) }) } } +#[rstest] #[tokio::test] -async fn token_source_failure_skips_http() { - let server = MockServer::start().await; - let cache = GcsCache::with_token_source( - config(&server, None), - JsonCodec::::new(), - Arc::new(FailingTokenSource), - ) - .unwrap(); +async fn token_source_failure_skips_http(#[future(awt)] server: MockServer) { + let cache = support::cache_with_token(&server, None, Arc::new(FailingTokenSource)); + assert_eq!( + cache.get_cache("key", &context()).unwrap_err(), + Error::Unavailable + ); assert_eq!( cache - .get_cache("key", &ExactCacheContext::default()) + .async_set_cache("key", json!(1), context()) + .await .unwrap_err(), Error::Unavailable ); diff --git a/litellm-rust/crates/cache-gcs/tests/contract.rs b/litellm-rust/crates/cache-gcs/tests/contract.rs new file mode 100644 index 00000000000..5841d0db642 --- /dev/null +++ b/litellm-rust/crates/cache-gcs/tests/contract.rs @@ -0,0 +1,65 @@ +mod support; + +use litellm_cache::ExactCacheContext; +use litellm_cache_testing as contract; +use rstest::{fixture, rstest}; +use serde_json::json; +use support::{FakeBucket, JsonGcsCache}; +use wiremock::MockServer; + +struct Gcs { + cache: JsonGcsCache, + _server: MockServer, +} + +#[fixture] +async fn gcs() -> Gcs { + let server = FakeBucket::serve().await; + Gcs { + cache: support::cache(&server, Some("contract")), + _server: server, + } +} + +#[fixture] +fn context() -> ExactCacheContext { + ExactCacheContext::default() +} + +const PREFIX: &str = "contract:"; + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn hit_and_miss(#[future(awt)] gcs: Gcs, context: ExactCacheContext) { + contract::hit_and_miss(&gcs.cache, context, PREFIX, json!({"answer": 42})).await; +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn sync_async_equivalence(#[future(awt)] gcs: Gcs, context: ExactCacheContext) { + contract::sync_async_equivalence(&gcs.cache, context, PREFIX, json!("first"), json!([2])).await; +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn overwrite_replaces(#[future(awt)] gcs: Gcs, context: ExactCacheContext) { + contract::overwrite_replaces(&gcs.cache, context, PREFIX, json!(1), json!({"b": 2})).await; +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn pipeline_writes_every_entry(#[future(awt)] gcs: Gcs, context: ExactCacheContext) { + contract::pipeline_writes_every_entry( + &gcs.cache, + context, + PREFIX, + vec![json!("a"), json!(2), json!({"c": true})], + ) + .await; +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn batch_preserves_order(#[future(awt)] gcs: Gcs, context: ExactCacheContext) { + contract::batch_preserves_order(&gcs.cache, context, PREFIX, json!("first"), json!(2)).await; +} diff --git a/litellm-rust/crates/cache-gcs/tests/support/mod.rs b/litellm-rust/crates/cache-gcs/tests/support/mod.rs new file mode 100644 index 00000000000..6097f0ee1bd --- /dev/null +++ b/litellm-rust/crates/cache-gcs/tests/support/mod.rs @@ -0,0 +1,87 @@ +#![allow(dead_code)] + +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +use litellm_cache::JsonCodec; +use litellm_cache_gcs::{GcsCache, GcsConfig, StaticTokenSource, TokenSource}; +use percent_encoding::percent_decode_str; +use serde_json::Value; +use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate, http::Method, matchers::any}; + +pub type JsonGcsCache = GcsCache>; + +pub fn config(server: &MockServer, gcs_path: Option<&str>) -> GcsConfig { + GcsConfig { + bucket_name: "bucket".into(), + gcs_path: gcs_path.map(str::to_string), + path_service_account: None, + endpoint: server.uri(), + } +} + +pub fn cache_with_token( + server: &MockServer, + gcs_path: Option<&str>, + token: Arc, +) -> JsonGcsCache { + GcsCache::with_token_source( + config(server, gcs_path), + reqwest::Client::new(), + JsonCodec::new(), + token, + ) +} + +pub fn cache(server: &MockServer, gcs_path: Option<&str>) -> JsonGcsCache { + cache_with_token(server, gcs_path, Arc::new(StaticTokenSource("tok".into()))) +} + +/// An in-memory bucket speaking the JSON API's media upload and `alt=media` download. +#[derive(Clone, Default)] +pub struct FakeBucket { + objects: Arc>>>, +} + +impl FakeBucket { + pub async fn serve() -> MockServer { + let server = MockServer::start().await; + Mock::given(any()) + .respond_with(Self::default()) + .mount(&server) + .await; + server + } +} + +impl Respond for FakeBucket { + fn respond(&self, request: &Request) -> ResponseTemplate { + let mut objects = self.objects.lock().unwrap(); + match request.method { + Method::POST => { + let name = request + .url + .query_pairs() + .find_map(|(key, value)| (key == "name").then(|| value.into_owned())) + .expect("uploads carry the object name"); + objects.insert(name, request.body.clone()); + ResponseTemplate::new(200) + } + Method::GET => { + let encoded = request + .url + .path() + .strip_prefix("/storage/v1/b/bucket/o/") + .expect("downloads address an object"); + let name = percent_decode_str(encoded).decode_utf8().unwrap(); + match objects.get(name.as_ref()) { + Some(body) => ResponseTemplate::new(200).set_body_bytes(body.clone()), + None => ResponseTemplate::new(404), + } + } + _ => ResponseTemplate::new(405), + } + } +} diff --git a/litellm-rust/crates/cache-memory/Cargo.toml b/litellm-rust/crates/cache-memory/Cargo.toml index 86ab01564c8..88124f5401e 100644 --- a/litellm-rust/crates/cache-memory/Cargo.toml +++ b/litellm-rust/crates/cache-memory/Cargo.toml @@ -9,6 +9,6 @@ repository.workspace = true litellm-cache.workspace = true [dev-dependencies] -serde_json.workspace = true +litellm-cache-testing.workspace = true rstest.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/cache-memory/src/cache.rs b/litellm-rust/crates/cache-memory/src/cache.rs index 85850c1d925..74476559cbe 100644 --- a/litellm-rust/crates/cache-memory/src/cache.rs +++ b/litellm-rust/crates/cache-memory/src/cache.rs @@ -7,8 +7,8 @@ use std::{ }; use litellm_cache::{ - BaseCache, BatchCache, CacheConnectionResult, CacheConnectionStatus, ClaimCache, CounterCache, - DeleteCache, Error, ExactCacheContext, FlushCache, IncrementOperation, SetCache, TtlCache, + BaseCache, BatchCache, ClaimCache, CounterCache, DeleteCache, DisconnectCache, Error, + ExactCacheContext, FlushCache, SetCache, TtlCache, }; const DEFAULT_MAX_SIZE_IN_MEMORY: usize = 200; @@ -75,7 +75,9 @@ impl InMemoryCache { expiration_heap: BinaryHeap::new(), }), max_size_in_memory: max_size_in_memory.unwrap_or(DEFAULT_MAX_SIZE_IN_MEMORY), - default_ttl: default_ttl.unwrap_or(DEFAULT_TTL), + default_ttl: default_ttl + .filter(|ttl| !ttl.is_zero()) + .unwrap_or(DEFAULT_TTL), max_entry_bytes, measure_value, now: Arc::new(now), @@ -91,21 +93,9 @@ impl InMemoryCache { if self.max_size_in_memory == 0 { return Ok(CacheWrite::Disabled); } - if let (Some(limit), Some(measure)) = (self.max_entry_bytes, &self.measure_value) - && measure(&value)? > limit - { - return Ok(CacheWrite::TooLarge); - } let now = (self.now)(); let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; - let key = key.into(); - Self::evict(&mut state, self.max_size_in_memory, now, &key); - let expiration = state.expirations.get(&key).copied(); - if expiration.is_none_or(|expiration| expiration < now) { - Self::set_expiration(&mut state, &key, now + ttl.unwrap_or(self.default_ttl)); - } - state.values.insert(key, value); - Ok(CacheWrite::Stored) + self.store(&mut state, key.into(), value, ttl, now) } pub fn get_cache(&self, key: &str) -> Result, Error> { @@ -121,6 +111,70 @@ impl InMemoryCache { Ok(state.values.get(key).cloned()) } + /// `check_value_size`: whether `value` fits `max_entry_bytes`. Always `true` without a + /// limit and a measure, since typed values have no generic size. + pub fn check_value_size(&self, value: &V) -> Result { + match (self.max_entry_bytes, &self.measure_value) { + (Some(limit), Some(measure)) => Ok(measure(value)? <= limit), + _ => Ok(true), + } + } + + /// `evict_cache`: drops expired entries, then the earliest-expiring ones until a new key + /// fits. + pub fn evict_cache(&self) -> Result<(), Error> { + let now = (self.now)(); + let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; + Self::evict(&mut state, self.max_size_in_memory, now, None); + Ok(()) + } + + /// `evict_element_if_expired`: `true` when `key` had expired and was removed. + pub fn evict_element_if_expired(&self, key: &str) -> Result { + let now = (self.now)(); + let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; + let expired = state + .expirations + .get(key) + .is_some_and(|expiration| *expiration < now); + if expired { + Self::remove(&mut state, key); + } + Ok(expired) + } + + /// `allow_ttl_override`: a write may set the TTL when the key has none or it has passed. + pub fn allow_ttl_override(&self, key: &str) -> Result { + let now = (self.now)(); + Ok(self + .expires_at(key)? + .is_none_or(|expiration| expiration < now)) + } + + /// The number of stored entries, expired ones included until they are evicted. + pub fn len(&self) -> Result { + Ok(self + .state + .lock() + .map_err(|_| Error::Unavailable)? + .values + .len()) + } + + pub fn is_empty(&self) -> Result { + Ok(self.len()? == 0) + } + + /// Entries in the expiration heap, stale ones included; bounded by eviction. + pub fn expiration_heap_len(&self) -> Result { + Ok(self + .state + .lock() + .map_err(|_| Error::Unavailable)? + .expiration_heap + .len()) + } + pub fn max_size_in_memory(&self) -> usize { self.max_size_in_memory } @@ -172,7 +226,9 @@ impl InMemoryCache { Ok(()) } - fn evict(state: &mut CacheState, capacity: usize, now: Duration, key: &str) { + /// Writing an existing `key` never evicts another entry, unlike Python, which pops the + /// earliest-expiring entry whenever the cache is full. + fn evict(state: &mut CacheState, capacity: usize, now: Duration, key: Option<&str>) { while let Some(Reverse((expiration, key))) = state.expiration_heap.peek().cloned() { if state.expirations.get(&key).copied() != Some(expiration) { state.expiration_heap.pop(); @@ -183,7 +239,7 @@ impl InMemoryCache { break; } } - if state.values.contains_key(key) { + if key.is_some_and(|key| state.values.contains_key(key)) { return; } while state.values.len() >= capacity { @@ -209,6 +265,40 @@ impl InMemoryCache { state.values.remove(key); state.expirations.remove(key); } + + /// `get_cache` under the held lock: an expired entry is removed and reads as missing. + fn live(state: &mut CacheState, key: &str, now: Duration) -> Option { + if state + .expirations + .get(key) + .is_some_and(|expiration| *expiration < now) + { + Self::remove(state, key); + } + state.values.get(key).cloned() + } + + /// Python `set_cache` under the held lock: evict first (even when `key` already exists), + /// then skip oversized values, then write, keeping a live key's expiry. + fn store( + &self, + state: &mut CacheState, + key: String, + value: V, + ttl: Option, + now: Duration, + ) -> Result { + Self::evict(state, self.max_size_in_memory, now, None); + if !self.check_value_size(&value)? { + return Ok(CacheWrite::TooLarge); + } + let expiration = state.expirations.get(&key).copied(); + if expiration.is_none_or(|expiration| expiration < now) { + Self::set_expiration(state, &key, now + ttl.unwrap_or(self.default_ttl)); + } + state.values.insert(key, value); + Ok(CacheWrite::Stored) + } } impl ClaimCache for InMemoryCache @@ -227,7 +317,7 @@ where } let now = (self.now)(); let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; - Self::evict(&mut state, self.max_size_in_memory, now, key); + Self::evict(&mut state, self.max_size_in_memory, now, Some(key)); let existing = state .values .get(key) @@ -262,38 +352,12 @@ impl CounterCache for InMemoryCache { } let now = (self.now)(); let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; - Self::evict(&mut state, self.max_size_in_memory, now, key); - let value = state.values.get(key).copied().unwrap_or_default() + amount; - if !state.expirations.contains_key(key) { - Self::set_expiration( - &mut state, - key, - now + self.get_ttl(&context).unwrap_or(self.default_ttl), - ); - } - state.values.insert(key.into(), value); + let value = Self::live(&mut state, key, now).unwrap_or_default() + amount; + self.store(&mut state, key.into(), value, self.get_ttl(&context), now)?; Ok(value) } } -impl InMemoryCache { - pub async fn async_increment_pipeline( - &self, - operations: Vec, - ) -> Result, Error> { - operations - .into_iter() - .map(|operation| { - self.increment_cache( - &operation.key, - operation.amount, - ExactCacheContext { ttl: operation.ttl }, - ) - }) - .collect() - } -} - impl BaseCache for InMemoryCache { type Value = V; type Context = ExactCacheContext; @@ -315,18 +379,12 @@ impl BaseCache for InMemoryCache { fn get_cache(&self, key: &str, _: &ExactCacheContext) -> Result, Error> { self.get_cache(key) } +} +impl DisconnectCache for InMemoryCache { async fn disconnect(&self) -> Result<(), Error> { Ok(()) } - - async fn test_connection(&self) -> Result { - Ok(CacheConnectionResult { - status: CacheConnectionStatus::Success, - message: "In-memory cache connection test successful".into(), - error: None, - }) - } } impl BatchCache for InMemoryCache {} @@ -367,34 +425,9 @@ where } let now = (self.now)(); let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; - Self::evict(&mut state, self.max_size_in_memory, now, key); - let mut stored = state.values.get(key).cloned().unwrap_or_default(); + let mut stored = Self::live(&mut state, key, now).unwrap_or_default(); stored.extend(values.iter().cloned()); - if let (Some(limit), Some(measure)) = (self.max_entry_bytes, &self.measure_value) - && measure(&stored)? > limit - { - return Ok(values); - } - if !state.expirations.contains_key(key) { - Self::set_expiration(&mut state, key, now + ttl.unwrap_or(self.default_ttl)); - } - state.values.insert(key.into(), stored); + self.store(&mut state, key.into(), stored, ttl, now)?; Ok(values) } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn repeated_increments_keep_one_heap_entry_per_expiration() { - let cache = InMemoryCache::::new(Some(4), None); - for _ in 0..100 { - cache - .increment_cache("counter", 1.0, ExactCacheContext::default()) - .unwrap(); - } - assert_eq!(cache.state.lock().unwrap().expiration_heap.len(), 1); - } -} diff --git a/litellm-rust/crates/cache-memory/tests/cache.rs b/litellm-rust/crates/cache-memory/tests/cache.rs index 0df0319b990..ce7d6ac8a98 100644 --- a/litellm-rust/crates/cache-memory/tests/cache.rs +++ b/litellm-rust/crates/cache-memory/tests/cache.rs @@ -8,111 +8,296 @@ use std::{ }; use litellm_cache::{ - BaseCache, CacheBackend, CacheConnectionStatus, ClaimCache, CounterCache, DeleteCache, Error, - ExactCacheContext, IncrementOperation, SetCache, get_cache, set_cache, + BaseCache, BatchCache, BatchEntry, CacheBackend, ClaimCache, CounterCache, DeleteCache, + DisconnectCache, Error, ExactCacheContext, FlushCache, IncrementOperation, SetCache, TtlCache, + get_cache, set_cache, }; use litellm_cache_memory::{CacheWrite, InMemoryCache}; use rstest::{fixture, rstest}; +type Clock = Arc; + #[fixture] -fn clock() -> Arc { +fn clock() -> Clock { Arc::new(AtomicU64::new(100)) } -fn cache(clock: Arc, capacity: usize) -> InMemoryCache { +fn cache_with(clock: &Clock, capacity: usize) -> InMemoryCache { + let clock = clock.clone(); InMemoryCache::with_clock(Some(capacity), Some(Duration::from_secs(60)), move || { - Duration::from_secs(clock.load(Ordering::SeqCst)) + Duration::from_millis(clock.load(Ordering::SeqCst) * 1000) }) } +fn cache(clock: &Clock, capacity: usize) -> InMemoryCache { + cache_with(clock, capacity) +} + +fn at(clock: &Clock, seconds: u64) { + clock.store(seconds, Ordering::SeqCst); +} + +fn secs(seconds: u64) -> Option { + Some(Duration::from_secs(seconds)) +} + +fn ttl(seconds: u64) -> ExactCacheContext { + ExactCacheContext { ttl: secs(seconds) } +} + +fn measured(capacity: usize) -> InMemoryCache { + InMemoryCache::with_clock_and_size_measurement( + Some(capacity), + secs(60), + Some(4), + Some(Arc::new(|value: &String| { + if value.is_empty() { + return Err(Error::InvalidEntry); + } + Ok(value.len()) + })), + || Duration::from_secs(100), + ) +} + #[rstest] -fn default_explicit_and_override_ttls_follow_python_rules(clock: Arc) { - let cache = cache(clock.clone(), 4); +fn default_explicit_and_override_ttls_follow_python_rules(clock: Clock) { + let cache = cache(&clock, 4); cache.set_cache("key", "first".into(), None).unwrap(); - assert_eq!( - cache.expires_at("key").unwrap(), - Some(Duration::from_secs(160)) - ); - cache - .set_cache("key", "second".into(), Some(Duration::from_secs(10))) - .unwrap(); - assert_eq!( - cache.expires_at("key").unwrap(), - Some(Duration::from_secs(160)) - ); - clock.store(160, Ordering::SeqCst); + assert_eq!(cache.expires_at("key").unwrap(), secs(160)); + cache.set_cache("key", "second".into(), secs(10)).unwrap(); + assert_eq!(cache.expires_at("key").unwrap(), secs(160)); + at(&clock, 160); assert_eq!(cache.get_cache("key").unwrap(), Some("second".into())); - clock.store(161, Ordering::SeqCst); + at(&clock, 161); assert_eq!(cache.get_cache("key").unwrap(), None); - cache - .set_cache("key", "third".into(), Some(Duration::from_secs(10))) - .unwrap(); - assert_eq!( - cache.expires_at("key").unwrap(), - Some(Duration::from_secs(171)) - ); + assert_eq!(cache.expires_at("key").unwrap(), None); + cache.set_cache("key", "third".into(), secs(10)).unwrap(); + assert_eq!(cache.expires_at("key").unwrap(), secs(171)); } #[rstest] -fn write_at_expiry_boundary_refreshes_ttl(clock: Arc) { - let cache = cache(clock.clone(), 4); - cache - .set_cache("key", "first".into(), Some(Duration::from_secs(10))) - .unwrap(); - clock.store(110, Ordering::SeqCst); - cache - .set_cache("key", "second".into(), Some(Duration::from_secs(10))) - .unwrap(); - assert_eq!( - cache.expires_at("key").unwrap(), - Some(Duration::from_secs(120)) - ); - clock.store(115, Ordering::SeqCst); +#[case::unset(None, secs(600))] +#[case::zero_falls_back_like_python_or(Some(Duration::ZERO), secs(600))] +#[case::explicit(secs(5), secs(5))] +fn default_ttl_falls_back_to_ten_minutes( + #[case] default_ttl: Option, + #[case] expected: Option, +) { + let cache = InMemoryCache::::with_clock(None, default_ttl, || Duration::ZERO); + assert_eq!(cache.get_ttl(&ExactCacheContext::default()), expected); + cache.set_cache("key", "value".into(), None).unwrap(); + assert_eq!(cache.expires_at("key").unwrap(), expected); + assert_eq!(cache.max_size_in_memory(), 200); +} + +#[rstest] +fn write_at_expiry_boundary_refreshes_ttl(clock: Clock) { + let cache = cache(&clock, 4); + cache.set_cache("key", "first".into(), secs(10)).unwrap(); + at(&clock, 110); + cache.set_cache("key", "second".into(), secs(10)).unwrap(); + assert_eq!(cache.expires_at("key").unwrap(), secs(120)); + at(&clock, 115); assert_eq!(cache.get_cache("key").unwrap(), Some("second".into())); } #[rstest] -fn capacity_evicts_earliest_and_ignores_stale_heap_entries(clock: Arc) { - let cache = cache(clock, 2); - cache - .set_cache("early", "a".into(), Some(Duration::from_secs(10))) - .unwrap(); - cache - .set_cache("late", "b".into(), Some(Duration::from_secs(20))) - .unwrap(); +fn expired_key_without_a_read_allows_a_ttl_override(clock: Clock) { + let cache = cache(&clock, 4); + cache.set_cache("key", "first".into(), secs(1)).unwrap(); + assert_eq!(cache.allow_ttl_override("key"), Ok(false)); + at(&clock, 102); + assert_eq!(cache.allow_ttl_override("key"), Ok(true)); + cache.set_cache("key", "second".into(), secs(1)).unwrap(); + assert_eq!(cache.expires_at("key").unwrap(), secs(103)); + assert_eq!(cache.allow_ttl_override("missing"), Ok(true)); +} + +#[rstest] +fn capacity_evicts_earliest_and_ignores_stale_heap_entries(clock: Clock) { + let cache = cache(&clock, 2); + cache.set_cache("early", "a".into(), secs(10)).unwrap(); + cache.set_cache("late", "b".into(), secs(20)).unwrap(); cache.delete_cache("early").unwrap(); - cache - .set_cache("new", "c".into(), Some(Duration::from_secs(30))) - .unwrap(); + cache.set_cache("new", "c".into(), secs(30)).unwrap(); assert_eq!(cache.get_cache("late").unwrap(), Some("b".into())); - cache - .set_cache("last", "d".into(), Some(Duration::from_secs(40))) - .unwrap(); + cache.set_cache("last", "d".into(), secs(40)).unwrap(); assert_eq!(cache.get_cache("late").unwrap(), None); } -#[test] -fn disabled_size_limited_and_validated_writes_are_observable() { - let cache = |capacity| { - InMemoryCache::with_clock_and_size_measurement( - Some(capacity), - Some(Duration::from_secs(60)), - Some(4), - Some(Arc::new(|value: &String| { - if value.is_empty() { - return Err(Error::InvalidEntry); - } - Ok(value.len()) - })), - || Duration::from_secs(100), - ) - }; - let disabled = cache(0); +#[rstest] +fn max_size_is_respected_when_every_item_has_a_long_ttl(clock: Clock) { + let cache = cache(&clock, 3); + for index in 0..3 { + at(&clock, 100 + index); + cache + .set_cache( + format!("key_{index}"), + format!("value_{index}"), + secs(86_400), + ) + .unwrap(); + } + assert_eq!(cache.len(), Ok(3)); + cache + .set_cache("key_3", "value_3".into(), secs(86_400)) + .unwrap(); + assert_eq!(cache.len(), Ok(3)); + assert_eq!(cache.get_cache("key_0").unwrap(), None); + assert_eq!(cache.expires_at("key_0").unwrap(), None); + for key in ["key_1", "key_2", "key_3"] { + assert!(cache.get_cache(key).unwrap().is_some(), "{key}"); + } +} + +#[rstest] +fn expired_items_are_evicted_before_live_ones(clock: Clock) { + let cache = cache(&clock, 3); + cache.set_cache("expired_1", "1".into(), secs(1)).unwrap(); + cache.set_cache("expired_2", "2".into(), secs(1)).unwrap(); + cache + .set_cache("long_lived", "3".into(), secs(86_400)) + .unwrap(); + assert_eq!(cache.len(), Ok(3)); + at(&clock, 102); + cache + .set_cache("new_item", "4".into(), secs(86_400)) + .unwrap(); + assert_eq!(cache.len(), Ok(2)); + assert_eq!(cache.get_cache("long_lived").unwrap(), Some("3".into())); + assert_eq!(cache.get_cache("new_item").unwrap(), Some("4".into())); + for key in ["expired_1", "expired_2"] { + assert_eq!(cache.expires_at(key).unwrap(), None, "{key}"); + } +} + +#[rstest] +fn injected_clock_controls_expiry_and_eviction(clock: Clock) { + let cache = cache(&clock, 2); + at(&clock, 0); + cache + .set_cache("first", "original".into(), secs(10)) + .unwrap(); + at(&clock, 9); + cache.set_cache("second", "survivor".into(), None).unwrap(); + assert_eq!(cache.get_cache("first").unwrap(), Some("original".into())); + at(&clock, 11); + assert_eq!(cache.get_cache("first").unwrap(), None); + cache + .set_cache("third", "replacement".into(), None) + .unwrap(); + assert_eq!(cache.get_cache("second").unwrap(), Some("survivor".into())); + at(&clock, 70); + cache.set_cache("fourth", "new".into(), None).unwrap(); + assert_eq!(cache.get_cache("second").unwrap(), None); assert_eq!( - disabled.set_cache("a", "x".into(), None).unwrap(), + cache.get_cache("third").unwrap(), + Some("replacement".into()) + ); + assert_eq!(cache.get_cache("fourth").unwrap(), Some("new".into())); +} + +#[rstest] +fn rewriting_one_key_keeps_one_heap_entry(clock: Clock) { + let cache = cache(&clock, 10); + for index in 0..1_000 { + cache + .set_cache("hot_key", format!("value_{index}"), secs(60)) + .unwrap(); + } + assert_eq!(cache.expiration_heap_len(), Ok(1)); +} + +#[rstest] +fn repeated_increments_keep_one_heap_entry_per_expiration() { + let cache = InMemoryCache::::new(Some(4), None); + for _ in 0..100 { + cache + .increment_cache("counter", 1.0, ExactCacheContext::default()) + .unwrap(); + } + assert_eq!(cache.expiration_heap_len(), Ok(1)); +} + +#[rstest] +fn reinserting_expired_keys_below_capacity_prunes_the_heap(clock: Clock) { + let cache = cache(&clock, 200); + for cycle in 0..3 { + for index in 0..5 { + cache + .set_cache(format!("key_{index}"), format!("value_{cycle}"), secs(1)) + .unwrap(); + } + at(&clock, 100 + 2 * (cycle + 1)); + } + for index in 0..5 { + cache + .set_cache(format!("key_{index}"), "final".into(), secs(1)) + .unwrap(); + } + assert_eq!(cache.len(), Ok(5)); + assert_eq!(cache.expiration_heap_len(), Ok(5)); +} + +#[rstest] +fn evict_cache_drops_expired_entries_then_makes_room(clock: Clock) { + let cache = cache(&clock, 2); + assert_eq!(cache.is_empty(), Ok(true)); + cache.set_cache("short", "a".into(), secs(1)).unwrap(); + cache.set_cache("long", "b".into(), secs(50)).unwrap(); + at(&clock, 102); + cache.evict_cache().unwrap(); + assert_eq!(cache.len(), Ok(1)); + assert_eq!(cache.expires_at("short").unwrap(), None); + cache.set_cache("longer", "c".into(), secs(90)).unwrap(); + cache.evict_cache().unwrap(); + assert_eq!(cache.len(), Ok(1)); + assert_eq!(cache.get_cache("long").unwrap(), None); + assert_eq!(cache.get_cache("longer").unwrap(), Some("c".into())); +} + +#[rstest] +fn evict_element_if_expired_reports_removal(clock: Clock) { + let cache = cache(&clock, 4); + cache.set_cache("key", "value".into(), secs(10)).unwrap(); + assert_eq!(cache.evict_element_if_expired("key"), Ok(false)); + assert_eq!(cache.evict_element_if_expired("missing"), Ok(false)); + at(&clock, 110); + assert_eq!(cache.evict_element_if_expired("key"), Ok(false)); + at(&clock, 111); + assert_eq!(cache.evict_element_if_expired("key"), Ok(true)); + assert_eq!(cache.len(), Ok(0)); + assert_eq!(cache.expires_at("key").unwrap(), None); +} + +#[rstest] +#[case::fits("ok", Ok(true))] +#[case::at_limit("four", Ok(true))] +#[case::too_large("oversized", Ok(false))] +#[case::measure_error("", Err(Error::InvalidEntry))] +fn check_value_size_applies_the_entry_limit( + #[case] value: &str, + #[case] expected: Result, +) { + assert_eq!(measured(2).check_value_size(&value.to_string()), expected); +} + +#[rstest] +fn values_are_unbounded_without_a_measure() { + let cache = InMemoryCache::::default(); + assert_eq!(cache.max_entry_bytes(), None); + assert_eq!(cache.check_value_size(&"x".repeat(1 << 20)), Ok(true)); +} + +#[rstest] +fn disabled_size_limited_and_validated_writes_are_observable() { + assert_eq!( + measured(0).set_cache("a", "x".into(), None).unwrap(), CacheWrite::Disabled ); - let cache = cache(2); + let cache = measured(2); + assert_eq!(cache.max_entry_bytes(), Some(4)); assert_eq!( cache.set_cache("large", "oversized".into(), None).unwrap(), CacheWrite::TooLarge @@ -132,30 +317,21 @@ fn disabled_size_limited_and_validated_writes_are_observable() { assert_eq!(cache.get_cache("small").unwrap(), None); } +#[rstest] #[tokio::test] -async fn connection_test_matches_python_result_contract() { +async fn disconnect_is_a_no_op_that_keeps_entries() { let cache = InMemoryCache::::default(); - let result = BaseCache::test_connection(&cache).await.unwrap(); - assert_eq!(result.status, CacheConnectionStatus::Success); - assert_eq!(result.message, "In-memory cache connection test successful"); - assert_eq!(result.error, None); - assert_eq!( - serde_json::to_value(result).unwrap(), - serde_json::json!({ - "status": "success", - "message": "In-memory cache connection test successful" - }) - ); + cache.set_cache("key", "value".into(), None).unwrap(); + cache.disconnect().await.unwrap(); + assert_eq!(cache.get_cache("key").unwrap(), Some("value".into())); } +#[rstest] #[tokio::test] -async fn generic_consumers_share_typed_values_and_honor_expiration() { - let clock = clock(); - let cache: CacheBackend> = Arc::new(cache(clock.clone(), 4)); +async fn generic_consumers_share_typed_values_and_honor_expiration(clock: Clock) { + let cache: CacheBackend> = Arc::new(self::cache(&clock, 4)); let reader = Arc::clone(&cache); - let context = ExactCacheContext { - ttl: Some(Duration::from_secs(5)), - }; + let context = ttl(5); set_cache(cache.as_ref(), "sync", "first".into(), &context).unwrap(); assert_eq!( get_cache(reader.as_ref(), "sync", &context).unwrap(), @@ -181,7 +357,7 @@ async fn generic_consumers_share_typed_values_and_honor_expiration() { reader.async_get_cache("async", &context).await.unwrap(), None ); - clock.store(106, Ordering::SeqCst); + at(&clock, 106); assert_eq!(get_cache(reader.as_ref(), "sync", &context).unwrap(), None); assert_eq!( reader.async_get_cache("batch", &context).await.unwrap(), @@ -189,34 +365,95 @@ async fn generic_consumers_share_typed_values_and_honor_expiration() { ); } -#[test] -fn claims_are_atomic_and_refresh_eligible_winners() { - let clock = clock(); - let cache = InMemoryCache::with_clock(Some(4), Some(Duration::from_secs(60)), { - let clock = clock.clone(); - move || Duration::from_secs(clock.load(Ordering::SeqCst)) - }); - let context = ExactCacheContext { - ttl: Some(Duration::from_secs(10)), - }; +#[rstest] +#[case::context_ttl(ttl(5), secs(105))] +#[case::default_ttl(ExactCacheContext::default(), secs(160))] +#[tokio::test] +async fn pipeline_writes_use_the_context_ttl_or_the_default( + clock: Clock, + #[case] context: ExactCacheContext, + #[case] expected: Option, +) { + let cache = cache(&clock, 4); + cache + .async_set_cache_pipeline( + vec![("a".into(), "1".into()), ("b".into(), "2".into())], + context, + ) + .await + .unwrap(); + assert_eq!(cache.expires_at("a").unwrap(), expected); + assert_eq!(cache.expires_at("b").unwrap(), expected); +} + +#[rstest] +#[tokio::test] +async fn batch_reads_return_one_entry_per_key_and_drop_expired_ones(clock: Clock) { + let cache = cache(&clock, 4); + cache.set_cache("short", "a".into(), secs(1)).unwrap(); + cache.set_cache("long", "b".into(), secs(50)).unwrap(); + let keys = vec!["short".to_string(), "missing".into(), "long".into()]; + assert_eq!( + cache + .batch_get_cache(&keys, &ExactCacheContext::default()) + .unwrap(), + [ + BatchEntry::Hit("a".to_string()), + BatchEntry::Miss, + BatchEntry::Hit("b".into()), + ] + ); + at(&clock, 102); + assert_eq!( + cache + .async_batch_get_cache(keys, ExactCacheContext::default()) + .await + .unwrap(), + [ + BatchEntry::Miss, + BatchEntry::Miss, + BatchEntry::Hit("b".into()) + ] + ); +} + +#[rstest] +#[tokio::test] +async fn flush_clears_values_and_expirations(clock: Clock) { + let cache = cache(&clock, 4); + cache.set_cache("a", "1".into(), None).unwrap(); + cache.set_cache("b", "2".into(), None).unwrap(); + cache.flush_cache().unwrap(); + assert_eq!(cache.len(), Ok(0)); + assert_eq!(cache.expiration_heap_len(), Ok(0)); + cache.set_cache("c", "3".into(), None).unwrap(); + FlushCache::async_flush_cache(&cache).await.unwrap(); + assert_eq!(cache.is_empty(), Ok(true)); + assert_eq!( + cache.async_get_oldest_n_keys(5).await.unwrap(), + Vec::::new() + ); +} + +#[rstest] +fn claims_are_atomic_and_refresh_eligible_winners(clock: Clock) { + let cache = cache(&clock, 4); + let context = ttl(10); assert_eq!( cache .claim_cache("affinity", "first".to_string(), &[], context.clone()) .unwrap(), "first" ); - clock.store(103, Ordering::SeqCst); + at(&clock, 103); assert_eq!( cache .claim_cache("affinity", "second".to_string(), &[], context.clone()) .unwrap(), "first" ); - assert_eq!( - cache.expires_at("affinity").unwrap(), - Some(Duration::from_secs(110)) - ); - clock.store(105, Ordering::SeqCst); + assert_eq!(cache.expires_at("affinity").unwrap(), secs(110)); + at(&clock, 105); assert_eq!( cache .claim_cache( @@ -228,13 +465,10 @@ fn claims_are_atomic_and_refresh_eligible_winners() { .unwrap(), "first" ); - assert_eq!( - cache.expires_at("affinity").unwrap(), - Some(Duration::from_secs(115)) - ); + assert_eq!(cache.expires_at("affinity").unwrap(), secs(115)); } -#[test] +#[rstest] fn counters_increment_under_one_lock() { let cache = InMemoryCache::::default(); assert_eq!( @@ -250,44 +484,107 @@ fn counters_increment_under_one_lock() { } #[rstest] -fn rewriting_an_existing_key_at_capacity_keeps_other_entries(clock: Arc) { - let cache = cache(clock, 2); - cache - .set_cache("hot", "1".into(), Some(Duration::from_secs(10))) - .unwrap(); - cache - .set_cache("cold", "2".into(), Some(Duration::from_secs(20))) - .unwrap(); +fn concurrent_increments_are_atomic() { + let cache = Arc::new(InMemoryCache::::default()); + cache.set_cache("counter", 1000.0, None).unwrap(); + let threads = (0..8) + .map(|_| { + let cache = cache.clone(); + std::thread::spawn(move || { + cache + .increment_cache("counter", 1.0, ExactCacheContext::default()) + .unwrap() + }) + }) + .collect::>(); + for thread in threads { + thread.join().unwrap(); + } + assert_eq!(cache.get_cache("counter").unwrap(), Some(1008.0)); +} + +#[rstest] +#[case::window_semantics(false)] +#[case::refresh_ttl_is_ignored(true)] +#[tokio::test] +async fn async_increment_delegates_to_the_locked_sync_path( + clock: Clock, + #[case] refresh_ttl: bool, +) { + let cache = cache_with::(&clock, 4); + assert_eq!( + cache + .async_increment("counter", 2.0, ttl(10), refresh_ttl) + .await, + Ok(2.0) + ); + at(&clock, 105); + assert_eq!( + cache + .async_increment("counter", 3.0, ttl(10), refresh_ttl) + .await, + Ok(5.0) + ); + assert_eq!(cache.get_cache("counter").unwrap(), Some(5.0)); + assert_eq!(cache.expires_at("counter").unwrap(), secs(110)); +} + +#[rstest] +fn expired_counters_restart_from_zero_with_a_new_ttl(clock: Clock) { + let cache = cache_with::(&clock, 4); + cache.increment_cache("counter", 2.0, ttl(10)).unwrap(); + at(&clock, 111); + assert_eq!(cache.increment_cache("counter", 1.0, ttl(10)), Ok(1.0)); + assert_eq!(cache.expires_at("counter").unwrap(), secs(121)); +} + +/// Python `InMemoryCache.set_cache` runs `evict_cache()` before every insert, and step 2 evicts +/// the earliest expiry while `len(cache_dict) >= max_size_in_memory`, even when the key being +/// written already exists. +#[rstest] +fn overwriting_an_existing_key_at_capacity_evicts_the_earliest_expiry_like_python(clock: Clock) { + let cache = cache(&clock, 2); + cache.set_cache("hot", "1".into(), secs(10)).unwrap(); + cache.set_cache("cold", "2".into(), secs(20)).unwrap(); cache.set_cache("cold", "3".into(), None).unwrap(); - assert_eq!(cache.get_cache("hot").unwrap(), Some("1".into())); + + assert_eq!(cache.get_cache("hot").unwrap(), None); assert_eq!(cache.get_cache("cold").unwrap(), Some("3".into())); +} + +/// `claim_cache` has no Python counterpart; it never evicts another entry for a key it holds. +#[rstest] +fn claiming_an_existing_key_at_capacity_keeps_other_entries(clock: Clock) { + let cache = cache(&clock, 2); + cache.set_cache("hot", "1".into(), secs(10)).unwrap(); + cache.set_cache("cold", "2".into(), secs(20)).unwrap(); cache .claim_cache("cold", "4".into(), &[], ExactCacheContext::default()) .unwrap(); assert_eq!(cache.get_cache("hot").unwrap(), Some("1".into())); - - cache.set_cache("new", "5".into(), None).unwrap(); - assert_eq!(cache.get_cache("hot").unwrap(), None); - assert_eq!(cache.get_cache("cold").unwrap(), Some("3".into())); - assert_eq!(cache.get_cache("new").unwrap(), Some("5".into())); + assert_eq!(cache.get_cache("cold").unwrap(), Some("2".into())); } -#[test] -fn incrementing_an_existing_counter_at_capacity_keeps_every_counter() { - let cache = InMemoryCache::::new(Some(2), None); +/// Python `increment_cache` is `get_cache` then `set_cache`, so at capacity the write evicts +/// the earliest expiry first: equal expiries tie-break on the key, and the value read before +/// eviction is the one written back. +#[rstest] +fn incrementing_at_capacity_evicts_the_earliest_expiry_like_python(clock: Clock) { + let cache = cache_with::(&clock, 2); for key in ["a", "b", "a", "b"] { cache .increment_cache(key, 1.0, ExactCacheContext::default()) .unwrap(); } - assert_eq!(cache.get_cache("a").unwrap(), Some(2.0)); + assert_eq!(cache.get_cache("a").unwrap(), None); assert_eq!(cache.get_cache("b").unwrap(), Some(2.0)); } -#[test] -fn disabled_cache_does_not_retain_claims_or_counters() { +#[rstest] +#[tokio::test] +async fn disabled_cache_does_not_retain_claims_counters_or_sets() { let claims = InMemoryCache::::new(Some(0), None); assert_eq!( claims @@ -305,62 +602,128 @@ fn disabled_cache_does_not_retain_claims_or_counters() { 2.0 ); assert_eq!(counters.get_cache("key").unwrap(), None); + + let sets = InMemoryCache::>::new(Some(0), None); + assert_eq!( + sets.async_set_cache_sadd("key", vec!["a".into()], None) + .await + .unwrap(), + ["a"] + ); + assert_eq!(sets.get_cache("key").unwrap(), None); } +#[rstest] #[tokio::test] -async fn ttl_and_oldest_key_operations_use_the_stored_expirations() { - let clock = Arc::new(AtomicU64::new(100)); - let cache = cache(clock, 3); - cache - .set_cache("later", "2".into(), Some(Duration::from_secs(20))) - .unwrap(); - cache - .set_cache("first", "1".into(), Some(Duration::from_secs(10))) - .unwrap(); +async fn ttl_and_oldest_key_operations_use_the_stored_expirations(clock: Clock) { + let cache = cache(&clock, 3); + cache.set_cache("later", "2".into(), secs(20)).unwrap(); + cache.set_cache("first", "1".into(), secs(10)).unwrap(); + cache.set_cache("latest", "3".into(), secs(30)).unwrap(); + assert_eq!(cache.async_get_ttl("first").await.unwrap(), secs(110)); assert_eq!( - cache.async_get_ttl("first").await.unwrap(), - Some(Duration::from_secs(110)) + TtlCache::async_get_ttl(&cache, "later").await.unwrap(), + secs(120) ); assert_eq!(cache.async_get_oldest_n_keys(1).await.unwrap(), ["first"]); + assert_eq!( + cache.async_get_oldest_n_keys(10).await.unwrap(), + ["first", "later", "latest"] + ); + assert_eq!( + cache.async_get_oldest_n_keys(0).await.unwrap(), + Vec::::new() + ); assert_eq!(cache.async_get_ttl("missing").await.unwrap(), None); } +#[rstest] #[tokio::test] -async fn increment_pipeline_preserves_operation_order() { - let cache = InMemoryCache::::new(Some(3), None); +async fn increment_pipeline_preserves_operation_order(clock: Clock) { + let cache = cache_with::(&clock, 3); + let operation = |key: &str, amount, ttl| IncrementOperation { + key: key.into(), + amount, + ttl: secs(ttl), + }; assert_eq!( cache .async_increment_pipeline(vec![ - IncrementOperation { - key: "a".into(), - amount: 1.0, - ttl: Some(Duration::from_secs(10)), - }, - IncrementOperation { - key: "a".into(), - amount: 2.0, - ttl: Some(Duration::from_secs(20)), - }, + operation("a", 1.0, 10), + operation("b", 5.0, 30), + operation("a", 2.0, 20), ]) .await .unwrap(), - [1.0, 3.0] + [1.0, 5.0, 3.0] ); assert_eq!(cache.get_cache("a").unwrap(), Some(3.0)); + assert_eq!(cache.expires_at("a").unwrap(), secs(110)); + assert_eq!(cache.expires_at("b").unwrap(), secs(130)); + assert_eq!( + cache.async_increment_pipeline(Vec::new()).await.unwrap(), + Vec::::new() + ); } +#[rstest] #[tokio::test] -async fn set_capability_preserves_python_result_and_deduplicates_storage() { - let cache = InMemoryCache::>::new(None, None); - let inserted = vec!["a".into(), "a".into(), "b".into()]; +async fn set_capability_preserves_python_result_and_deduplicates_storage(clock: Clock) { + let cache = cache_with::>(&clock, 4); + let inserted = vec!["a".to_string(), "a".into(), "b".into()]; assert_eq!( cache - .async_set_cache_sadd("members", inserted.clone(), None) + .async_set_cache_sadd("members", inserted.clone(), secs(10)) .await .unwrap(), inserted ); + assert_eq!( + cache + .async_set_cache_sadd("members", vec!["c".into()], secs(99)) + .await + .unwrap(), + ["c"] + ); + assert_eq!( + cache.get_cache("members").unwrap(), + Some(HashSet::from(["a".into(), "b".into(), "c".into()])) + ); + assert_eq!(cache.expires_at("members").unwrap(), secs(110)); + at(&clock, 111); + cache + .async_set_cache_sadd("members", vec!["d".into()], None) + .await + .unwrap(); + assert_eq!( + cache.get_cache("members").unwrap(), + Some(HashSet::from(["d".into()])) + ); + assert_eq!(cache.expires_at("members").unwrap(), secs(171)); +} + +#[rstest] +#[tokio::test] +async fn oversized_set_additions_are_not_stored() { + let cache = InMemoryCache::>::with_clock_and_size_measurement( + Some(4), + None, + Some(2), + Some(Arc::new(|value: &HashSet| Ok(value.len()))), + || Duration::ZERO, + ); + cache + .async_set_cache_sadd("members", vec!["a".into(), "b".into()], None) + .await + .unwrap(); + assert_eq!( + cache + .async_set_cache_sadd("members", vec!["c".into()], None) + .await + .unwrap(), + ["c"] + ); assert_eq!( cache.get_cache("members").unwrap(), Some(HashSet::from(["a".into(), "b".into()])) diff --git a/litellm-rust/crates/cache-memory/tests/contract.rs b/litellm-rust/crates/cache-memory/tests/contract.rs new file mode 100644 index 00000000000..860de1ed798 --- /dev/null +++ b/litellm-rust/crates/cache-memory/tests/contract.rs @@ -0,0 +1,98 @@ +use std::time::Duration; + +use litellm_cache::ExactCacheContext; +use litellm_cache_memory::InMemoryCache; +use litellm_cache_testing as contract; +use rstest::{fixture, rstest}; + +#[fixture] +fn strings() -> InMemoryCache { + InMemoryCache::new(Some(16), None) +} + +#[fixture] +fn counters() -> InMemoryCache { + InMemoryCache::new(Some(16), None) +} + +#[fixture] +fn context() -> ExactCacheContext { + ExactCacheContext { + ttl: Some(Duration::from_secs(60)), + } +} + +#[rstest] +#[tokio::test] +async fn hit_and_miss(strings: InMemoryCache, context: ExactCacheContext) { + contract::hit_and_miss(&strings, context, "memory:", "value".into()).await; +} + +#[rstest] +#[tokio::test] +async fn sync_async_equivalence(strings: InMemoryCache, context: ExactCacheContext) { + contract::sync_async_equivalence( + &strings, + context, + "memory:", + "first".into(), + "second".into(), + ) + .await; +} + +#[rstest] +#[tokio::test] +async fn overwrite_replaces(strings: InMemoryCache, context: ExactCacheContext) { + contract::overwrite_replaces( + &strings, + context, + "memory:", + "first".into(), + "second".into(), + ) + .await; +} + +#[rstest] +#[tokio::test] +async fn pipeline_writes_every_entry(strings: InMemoryCache, context: ExactCacheContext) { + contract::pipeline_writes_every_entry( + &strings, + context, + "memory:", + vec!["a".into(), "b".into(), "c".into()], + ) + .await; +} + +#[rstest] +#[tokio::test] +async fn batch_preserves_order(strings: InMemoryCache, context: ExactCacheContext) { + contract::batch_preserves_order( + &strings, + context, + "memory:", + "first".into(), + "second".into(), + ) + .await; +} + +#[rstest] +#[tokio::test] +async fn delete_removes_key(strings: InMemoryCache, context: ExactCacheContext) { + contract::delete_removes_key(&strings, context, "memory:", "value".into()).await; +} + +#[rstest] +#[tokio::test] +async fn flush_clears(strings: InMemoryCache, context: ExactCacheContext) { + contract::flush_clears(&strings, context, "memory:", "value".into()).await; +} + +#[rstest] +#[tokio::test] +async fn counter_accumulates(counters: InMemoryCache, context: ExactCacheContext) { + contract::counter_accumulates(&counters, context, "memory:").await; +} diff --git a/litellm-rust/crates/cache-qdrant-semantic/Cargo.toml b/litellm-rust/crates/cache-qdrant-semantic/Cargo.toml index 09d6a9637f3..950c2db7491 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/Cargo.toml +++ b/litellm-rust/crates/cache-qdrant-semantic/Cargo.toml @@ -17,7 +17,8 @@ tokio.workspace = true uuid.workspace = true [dev-dependencies] -litellm-cache-response.workspace = true +futures-executor = "0.3" +litellm-cache-testing.workspace = true rstest.workspace = true tonic = "0.14" tonic-prost = "0.14" diff --git a/litellm-rust/crates/cache-qdrant-semantic/src/semantic.rs b/litellm-rust/crates/cache-qdrant-semantic/src/cache.rs similarity index 72% rename from litellm-rust/crates/cache-qdrant-semantic/src/semantic.rs rename to litellm-rust/crates/cache-qdrant-semantic/src/cache.rs index fb165ed5a8e..a140e0af174 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/src/semantic.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/src/cache.rs @@ -1,7 +1,8 @@ -use std::future::Future; - use futures_util::future::try_join_all; -use litellm_cache::{BaseCache, CacheCodec, CacheConnectionResult, Error, SemanticCacheContext}; +use litellm_cache::{ + BaseCache, CacheCodec, Error, SemanticCacheContext, + semantic::{Embedder, SemanticCache, SemanticLookup, prompt_from_messages}, +}; use qdrant_client::{ Payload, Qdrant, qdrant::{ @@ -14,26 +15,7 @@ use qdrant_client::{ use serde_json::{Map, Value, json}; use uuid::Uuid; -use crate::prompt_from_messages; - -pub trait Embedder: Send + Sync + 'static { - fn model(&self) -> &str; - fn embed(&self, input: &str) -> impl Future, Error>> + Send; -} - -#[derive(Clone, Debug, PartialEq)] -pub enum Quantization { - Binary, - Scalar, - Product, -} - -pub struct QdrantSemanticConfig { - pub collection_name: String, - pub similarity_threshold: f64, - pub vector_size: u64, - pub quantization: Quantization, -} +use crate::{QdrantSemanticConfig, Quantization}; pub struct QdrantSemanticCache { client: Qdrant, @@ -100,14 +82,9 @@ impl QdrantSemanticCache { &self.embedder } + /// Python reads `kwargs["messages"]` unguarded, so a request without messages fails. fn prompt(context: &SemanticCacheContext) -> Result { - let Some(messages) = context.messages.as_ref().and_then(Value::as_array) else { - return Err(Error::MissingPrompt); - }; - if messages.is_empty() { - return Err(Error::MissingPrompt); - } - Ok(prompt_from_messages(messages)) + prompt_from_messages(context).ok_or(Error::MissingPrompt) } async fn set( @@ -117,7 +94,10 @@ impl QdrantSemanticCache { context: &SemanticCacheContext, ) -> Result<(), Error> { let prompt = Self::prompt(context)?; - let vector = self.embedder.embed(&prompt).await?; + let vector = self + .embedder + .async_embed(&prompt, context.metadata.as_ref()) + .await?; let response = String::from_utf8(self.codec.encode(&value)?).map_err(|_| Error::InvalidEntry)?; let payload = Payload::try_from(json!({ @@ -147,9 +127,12 @@ impl QdrantSemanticCache { &self, key: &str, context: &SemanticCacheContext, - ) -> Result, Error> { + ) -> Result, Error> { let prompt = Self::prompt(context)?; - let vector = self.embedder.embed(&prompt).await?; + let vector = self + .embedder + .async_embed(&prompt, context.metadata.as_ref()) + .await?; let result = self .client .search_points( @@ -171,20 +154,27 @@ impl QdrantSemanticCache { .await .map_err(|_| Error::Unavailable)?; let Some(point) = result.result.into_iter().next() else { - return Ok(None); + return Ok(SemanticLookup::miss(Some(0.0))); }; let payload: Map = Payload::from(point.payload).into(); - if payload.get("litellm_cache_key").and_then(Value::as_str) != Some(key) { - return Ok(None); + if !payload + .get("litellm_cache_key") + .is_some_and(|cached| python_str(cached).as_deref() == Some(key)) + { + return Ok(SemanticLookup::miss(Some(0.0))); } - if f64::from(point.score) < self.config.similarity_threshold { - return Ok(None); + let similarity = f64::from(point.score); + if similarity < self.config.similarity_threshold { + return Ok(SemanticLookup::miss(Some(similarity))); } let response = payload .get("response") .and_then(Value::as_str) .ok_or(Error::InvalidEntry)?; - self.codec.decode(response.as_bytes()).map(Some) + Ok(SemanticLookup { + value: Some(self.codec.decode(response.as_bytes())?), + similarity: Some(similarity), + }) } } @@ -219,7 +209,8 @@ impl BaseCache for QdrantSemanticCache { } fn get_cache(&self, key: &str, context: &Self::Context) -> Result, Error> { - self.runtime.block_on(self.get(key, context)) + self.get_cache_with_similarity(key, context) + .map(|lookup| lookup.value) } async fn async_set_cache( @@ -236,7 +227,7 @@ impl BaseCache for QdrantSemanticCache { key: &str, context: &Self::Context, ) -> Result, Error> { - self.get(key, context).await + self.get(key, context).await.map(|lookup| lookup.value) } async fn async_set_cache_pipeline( @@ -251,12 +242,36 @@ impl BaseCache for QdrantSemanticCache { .await .map(|_| ()) } +} - async fn disconnect(&self) -> Result<(), Error> { - Ok(()) +/// Python stamps the top point's score, even below the threshold, and `0.0` when there is no +/// point or it belongs to another key. A request without messages fails before any search. +impl SemanticCache for QdrantSemanticCache { + fn get_cache_with_similarity( + &self, + key: &str, + context: &Self::Context, + ) -> Result, Error> { + self.runtime.block_on(self.get(key, context)) } - async fn test_connection(&self) -> Result { - Err(Error::UnsupportedOperation) + async fn async_get_cache_with_similarity( + &self, + key: &str, + context: &Self::Context, + ) -> Result, Error> { + self.get(key, context).await + } +} + +/// `str(value)` for the scalar payload values `_payload_matches_cache_key` compares; `None` for +/// null (a pre-isolation point without a key) and for containers, which never equal a key. +fn python_str(value: &Value) -> Option { + match value { + Value::String(text) => Some(text.clone()), + Value::Number(number) => Some(number.to_string()), + Value::Bool(true) => Some("True".into()), + Value::Bool(false) => Some("False".into()), + Value::Null | Value::Array(_) | Value::Object(_) => None, } } diff --git a/litellm-rust/crates/cache-qdrant-semantic/src/config.rs b/litellm-rust/crates/cache-qdrant-semantic/src/config.rs new file mode 100644 index 00000000000..2654c155f23 --- /dev/null +++ b/litellm-rust/crates/cache-qdrant-semantic/src/config.rs @@ -0,0 +1,13 @@ +#[derive(Clone, Debug, PartialEq)] +pub enum Quantization { + Binary, + Scalar, + Product, +} + +pub struct QdrantSemanticConfig { + pub collection_name: String, + pub similarity_threshold: f64, + pub vector_size: u64, + pub quantization: Quantization, +} diff --git a/litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs b/litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs index 47b898d6f4e..340393600f2 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs @@ -1,11 +1,9 @@ use std::time::Duration; -use litellm_cache::Error; +use litellm_cache::{Error, semantic::Embedder}; use reqwest::Client; use serde_json::Value; -use crate::Embedder; - pub struct OpenAiEmbedder { client: Client, api_base: String, @@ -31,14 +29,16 @@ impl OpenAiEmbedder { timeout: config.timeout, } } -} -impl Embedder for OpenAiEmbedder { - fn model(&self) -> &str { + pub fn model(&self) -> &str { &self.model } +} - async fn embed(&self, input: &str) -> Result, Error> { +/// An OpenAI-compatible `/embeddings` call. It has no router to route on, so `metadata` is +/// unused, and it only embeds asynchronously: sync cache calls block on the cache's runtime. +impl Embedder for OpenAiEmbedder { + async fn async_embed(&self, input: &str, _metadata: Option<&Value>) -> Result, Error> { let request = self .client .post(format!("{}/embeddings", self.api_base)) diff --git a/litellm-rust/crates/cache-qdrant-semantic/src/lib.rs b/litellm-rust/crates/cache-qdrant-semantic/src/lib.rs index 0f346a9155b..3017bc695e2 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/src/lib.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/src/lib.rs @@ -1,7 +1,7 @@ +mod cache; +mod config; mod embedder; -mod prompt; -mod semantic; +pub use cache::QdrantSemanticCache; +pub use config::{QdrantSemanticConfig, Quantization}; pub use embedder::{OpenAiEmbedder, OpenAiEmbedderConfig}; -pub use prompt::prompt_from_messages; -pub use semantic::{Embedder, QdrantSemanticCache, QdrantSemanticConfig, Quantization}; diff --git a/litellm-rust/crates/cache-qdrant-semantic/src/prompt.rs b/litellm-rust/crates/cache-qdrant-semantic/src/prompt.rs deleted file mode 100644 index ef1a2306658..00000000000 --- a/litellm-rust/crates/cache-qdrant-semantic/src/prompt.rs +++ /dev/null @@ -1,59 +0,0 @@ -use serde_json::Value; - -fn search_results_text(search_results: Option<&Value>) -> String { - let Some(Value::Array(results)) = search_results else { - return String::new(); - }; - results - .iter() - .filter_map(Value::as_object) - .flat_map(|result| { - let source = result - .get("source") - .and_then(Value::as_str) - .map(str::to_owned); - let title = result - .get("title") - .and_then(Value::as_str) - .map(str::to_owned); - let content = result - .get("content") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(Value::as_object) - .filter_map(|block| block.get("text").and_then(Value::as_str).map(str::to_owned)); - let citations = result - .get("citations") - .filter(|value| !value.is_null()) - .map(|value| serde_json::to_string(value).unwrap_or_default()); - source - .into_iter() - .chain(title) - .chain(content) - .chain(citations) - }) - .collect() -} - -pub fn prompt_from_messages(messages: &[Value]) -> String { - messages - .iter() - .filter_map(Value::as_object) - .map(|message| { - let content = match message.get("content") { - Some(Value::String(content)) => content.clone(), - Some(Value::Array(parts)) => parts - .iter() - .filter_map(Value::as_object) - .filter_map(|part| part.get("text").and_then(Value::as_str)) - .collect(), - _ => String::new(), - }; - format!( - "{content}{}", - search_results_text(message.get("search_results")) - ) - }) - .collect() -} diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/contract.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/contract.rs new file mode 100644 index 00000000000..896b799decf --- /dev/null +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/contract.rs @@ -0,0 +1,91 @@ +//! `overwrite_replaces` does not apply: like Python, every write upserts a new `uuid4` point, +//! so a second write with the same prompt adds a tie instead of replacing the first. + +mod support; + +use std::future::Future; + +use litellm_cache::{JsonCodec, SemanticCacheContext, semantic::PreparedEmbedding}; +use litellm_cache_qdrant_semantic::{QdrantSemanticCache, QdrantSemanticConfig, Quantization}; +use litellm_cache_testing as contract; +use qdrant_client::Qdrant; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; +use support::{FakeQdrant, FakeState}; + +type Cache = QdrantSemanticCache>; + +const PREFIX: &str = "contract:"; + +#[fixture] +fn context() -> SemanticCacheContext { + SemanticCacheContext { + messages: Some(json!([{"role": "user", "content": "contract prompt"}])), + ..Default::default() + } +} + +/// Runs a contract against a fresh fake Qdrant. The sync cache methods block on the runtime, so +/// the contract is polled on a blocking thread outside the runtime's own executor. +async fn run(check: F) +where + F: FnOnce(Cache) -> Fut + Send + 'static, + Fut: Future, +{ + let server = FakeQdrant::start(FakeState::default()).await; + let runtime = tokio::runtime::Handle::current(); + let cache = QdrantSemanticCache::connect( + Qdrant::from_url(&server.url()).build().unwrap(), + PreparedEmbedding(vec![0.6, 0.8]), + JsonCodec::new(), + QdrantSemanticConfig { + collection_name: "contract".to_owned(), + similarity_threshold: 0.9, + vector_size: 2, + quantization: Quantization::Binary, + }, + runtime.clone(), + ) + .await + .unwrap(); + tokio::task::spawn_blocking(move || { + let _guard = runtime.enter(); + futures_executor::block_on(check(cache)); + }) + .await + .unwrap(); + server.stop(); +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn hit_and_miss(context: SemanticCacheContext) { + run(|cache| async move { + contract::hit_and_miss(&cache, context, PREFIX, json!({"answer": 42})).await; + }) + .await; +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn sync_async_equivalence(context: SemanticCacheContext) { + run(|cache| async move { + contract::sync_async_equivalence(&cache, context, PREFIX, json!("first"), json!([2])).await; + }) + .await; +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn pipeline_writes_every_entry(context: SemanticCacheContext) { + run(|cache| async move { + contract::pipeline_writes_every_entry( + &cache, + context, + PREFIX, + vec![json!("a"), json!(2), json!({"c": true})], + ) + .await; + }) + .await; +} diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs index 6b09448fde8..de0fab0a66f 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs @@ -3,9 +3,10 @@ use std::{ time::Duration, }; -use litellm_cache::Error; -use litellm_cache_qdrant_semantic::{Embedder, OpenAiEmbedder, OpenAiEmbedderConfig}; -use serde_json::Value; +use litellm_cache::{Error, semantic::Embedder}; +use litellm_cache_qdrant_semantic::{OpenAiEmbedder, OpenAiEmbedderConfig}; +use rstest::rstest; +use serde_json::{Value, json}; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::TcpListener, @@ -98,6 +99,7 @@ fn config(base: String, timeout: Option) -> OpenAiEmbedderConfig { } } +#[rstest] #[tokio::test] async fn posts_embeddings_request_and_parses_vector() { let server = TestHttpServer::response("200 OK", r#"{"data":[{"embedding":[0.1,0.2]}]}"#).await; @@ -108,7 +110,14 @@ async fn posts_embeddings_request_and_parses_vector() { Some(Duration::from_secs(1)), ), ); - assert_eq!(embedder.embed("hello").await.unwrap(), vec![0.1, 0.2]); + assert_eq!(embedder.model(), "test-model"); + assert_eq!( + embedder + .async_embed("hello", Some(&json!({"ignored": true}))) + .await + .unwrap(), + vec![0.1, 0.2] + ); let request = server.request.lock().unwrap().clone().unwrap(); let request_text = String::from_utf8(request).unwrap(); assert!(request_text.starts_with("POST /embeddings HTTP/1.1\r\n")); @@ -120,37 +129,50 @@ async fn posts_embeddings_request_and_parses_vector() { assert_eq!(body["encoding_format"], "float"); } +#[rstest] +#[case::error_status("500 Internal Server Error", "{}", 0, None, Err(Error::Unavailable))] +#[case::timed_out( + "200 OK", + r#"{"data":[{"embedding":[0.1,0.2]}]}"#, + 500, + Some(Duration::from_millis(200)), + Err(Error::Unavailable) +)] +#[case::within_timeout( + "200 OK", + r#"{"data":[{"embedding":[0.1,0.2]}]}"#, + 100, + Some(Duration::from_secs(1)), + Ok(vec![0.1, 0.2]) +)] +#[case::missing_embedding("200 OK", r#"{"data":[]}"#, 0, None, Err(Error::Unavailable))] #[tokio::test] -async fn status_and_timeout_errors_are_unavailable() { - let server = TestHttpServer::response("500 Internal Server Error", "{}").await; - let embedder = OpenAiEmbedder::new(reqwest::Client::new(), config(server.base_url(), None)); - assert_eq!(embedder.embed("hello").await, Err(Error::Unavailable)); - - let server = TestHttpServer::response_after( - "200 OK", - r#"{"data":[{"embedding":[0.1,0.2]}]}"#, - Duration::from_millis(500), - ) - .await; - let embedder = OpenAiEmbedder::new( - reqwest::Client::new(), - config(server.base_url(), Some(Duration::from_millis(200))), - ); - assert_eq!(embedder.embed("hello").await, Err(Error::Unavailable)); - - let server = TestHttpServer::response_after( - "200 OK", - r#"{"data":[{"embedding":[0.1,0.2]}]}"#, - Duration::from_millis(100), - ) - .await; - let embedder = OpenAiEmbedder::new( - reqwest::Client::new(), - config(server.base_url(), Some(Duration::from_secs(1))), - ); - assert_eq!(embedder.embed("hello").await.unwrap(), vec![0.1, 0.2]); +async fn status_timeout_and_body_errors_are_unavailable( + #[case] status: &str, + #[case] body: &str, + #[case] delay_ms: u64, + #[case] timeout: Option, + #[case] expected: Result, Error>, +) { + let server = + TestHttpServer::response_after(status, body, Duration::from_millis(delay_ms)).await; + let embedder = OpenAiEmbedder::new(reqwest::Client::new(), config(server.base_url(), timeout)); + assert_eq!(embedder.async_embed("hello", None).await, expected); } +#[rstest] +fn sync_embedding_is_unsupported() { + let embedder = OpenAiEmbedder::new( + reqwest::Client::new(), + config("http://127.0.0.1:9".to_owned(), None), + ); + assert_eq!( + embedder.embed("hello", None), + Err(Error::UnsupportedOperation) + ); +} + +#[rstest] #[tokio::test] async fn uses_the_injected_client() { let server = TestHttpServer::response("200 OK", r#"{"data":[{"embedding":[0.1,0.2]}]}"#).await; @@ -159,7 +181,10 @@ async fn uses_the_injected_client() { .build() .unwrap(); let embedder = OpenAiEmbedder::new(client, config(server.base_url(), None)); - assert_eq!(embedder.embed("hello").await.unwrap(), vec![0.1, 0.2]); + assert_eq!( + embedder.async_embed("hello", None).await.unwrap(), + vec![0.1, 0.2] + ); let request = server.request.lock().unwrap().clone().unwrap(); let request_text = String::from_utf8(request).unwrap(); assert!(request_text.contains("\r\nuser-agent: litellm-embedder-test\r\n")); diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/prompt.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/prompt.rs deleted file mode 100644 index 38cd9e2f908..00000000000 --- a/litellm-rust/crates/cache-qdrant-semantic/tests/prompt.rs +++ /dev/null @@ -1,38 +0,0 @@ -use litellm_cache_qdrant_semantic::prompt_from_messages; -use serde_json::json; - -#[test] -fn prompt_matches_python_message_content_rules() { - let messages = vec![ - json!({"role": "user", "content": "hello"}), - json!({ - "role": "user", - "content": [ - {"type": "text", "text": "world"}, - {"type": "image_url", "image_url": {"url": "ignored"}}, - {"type": "text", "text": "!"}, - ], - }), - ]; - - assert_eq!(prompt_from_messages(&messages), "helloworld!"); -} - -#[test] -fn prompt_includes_search_result_text_and_compact_citations() { - let messages = vec![json!({ - "role": "tool", - "content": null, - "search_results": [{ - "source": "source", - "title": "title", - "content": [{"text": "body"}], - "citations": {"page": 1, "section": "intro"}, - }], - })]; - - assert_eq!( - prompt_from_messages(&messages), - r#"sourcetitlebody{"page":1,"section":"intro"}"# - ); -} diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs index c7522c0b313..fe25c503a36 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs @@ -1,27 +1,32 @@ -#[path = "support/mod.rs"] mod support; -use std::{collections::HashMap, sync::Arc, time::Duration}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::Duration, +}; -use litellm_cache::{BaseCache, CacheCodec, CacheContext, Error, SemanticCacheContext}; -use litellm_cache_qdrant_semantic::{ - Embedder, QdrantSemanticCache, QdrantSemanticConfig, Quantization, +use litellm_cache::{ + BaseCache, CacheContext, Error, JsonCodec, SemanticCacheContext, + semantic::{Embedder, SemanticCache, SemanticLookup}, }; -use litellm_cache_response::{ - CacheEntry, CacheKeyInput, ResponseCache, ResponseCacheCodec, ResponseCacheRequest, -}; -use qdrant_client::Payload; +use litellm_cache_qdrant_semantic::{QdrantSemanticCache, QdrantSemanticConfig, Quantization}; use qdrant_client::{ - Qdrant, + Payload, Qdrant, qdrant::{self, CompressionRatio, Distance, PointId, QuantizationType, Value, VectorParams}, }; +use rstest::{fixture, rstest}; use serde_json::{Value as JsonValue, json}; - use support::{FakeQdrant, FakeState, StoredPoint}; +type Calls = Arc)>>>; +type Cache = QdrantSemanticCache>; + +/// Embeds known prompts, fails on anything else, and records every call. #[derive(Clone)] struct FixedEmbedder { vectors: Arc>>, + calls: Calls, } impl FixedEmbedder { @@ -33,16 +38,21 @@ impl FixedEmbedder { .map(|(prompt, vector)| (prompt.to_owned(), vector)) .collect(), ), + calls: Calls::default(), } } } impl Embedder for FixedEmbedder { - fn model(&self) -> &str { - "fixed" - } - - async fn embed(&self, input: &str) -> Result, Error> { + async fn async_embed( + &self, + input: &str, + metadata: Option<&JsonValue>, + ) -> Result, Error> { + self.calls + .lock() + .unwrap() + .push((input.to_owned(), metadata.cloned())); self.vectors.get(input).cloned().ok_or(Error::Unavailable) } } @@ -63,22 +73,20 @@ fn context(prompt: &str) -> SemanticCacheContext { } } -fn value(response: JsonValue) -> CacheEntry { - CacheEntry { - timestamp: Some(1.0), - response, - } +#[fixture] +fn entry() -> JsonValue { + json!({"timestamp": 1.0, "response": {"answer": 42}}) } async fn connect( server: &FakeQdrant, vectors: impl IntoIterator)>, -) -> QdrantSemanticCache { +) -> Cache { let client = Qdrant::from_url(&server.url()).build().unwrap(); QdrantSemanticCache::connect( client, FixedEmbedder::new(vectors), - ResponseCacheCodec, + JsonCodec::new(), config(Quantization::Binary), tokio::runtime::Handle::current(), ) @@ -86,72 +94,70 @@ async fn connect( .unwrap() } +#[rstest] +#[case::binary(Quantization::Binary)] +#[case::scalar(Quantization::Scalar)] +#[case::product(Quantization::Product)] #[tokio::test(flavor = "multi_thread")] -#[expect( - deprecated, - reason = "the test verifies Qdrant's legacy always_ram quantization contract" -)] -async fn connect_sets_collection_quantization_and_index() { - for (quantization, expected) in [ - (Quantization::Binary, 0), - (Quantization::Scalar, 1), - (Quantization::Product, 2), - ] { - let server = FakeQdrant::start(FakeState::default()).await; - let client = Qdrant::from_url(&server.url()).build().unwrap(); - QdrantSemanticCache::connect( - client, - FixedEmbedder::new([]), - ResponseCacheCodec, - config(quantization), - tokio::runtime::Handle::current(), - ) - .await +async fn connect_sets_collection_quantization_and_index(#[case] quantization: Quantization) { + let server = FakeQdrant::start(FakeState::default()).await; + let client = Qdrant::from_url(&server.url()).build().unwrap(); + QdrantSemanticCache::connect( + client, + FixedEmbedder::new([]), + JsonCodec::::new(), + config(quantization.clone()), + tokio::runtime::Handle::current(), + ) + .await + .unwrap(); + let state = server.state.lock().unwrap(); + let request = &state.created_collections[0]; + let Some(qdrant::vectors_config::Config::Params(VectorParams { size, distance, .. })) = request + .vectors_config + .as_ref() + .and_then(|config| config.config.clone()) + else { + panic!("missing vector params"); + }; + assert_eq!(size, 2); + assert_eq!(distance, Distance::Cosine as i32); + let quantization_config = request + .quantization_config + .as_ref() + .unwrap() + .quantization .unwrap(); - let state = server.state.lock().unwrap(); - let request = &state.created_collections[0]; - let Some(qdrant::vectors_config::Config::Params(VectorParams { size, distance, .. })) = - request - .vectors_config - .as_ref() - .and_then(|config| config.config.clone()) - else { - panic!("missing vector params"); - }; - assert_eq!(size, 2); - assert_eq!(distance, Distance::Cosine as i32); - let quantization_config = request - .quantization_config - .as_ref() - .unwrap() - .quantization - .unwrap(); - match (expected, quantization_config) { - (0, qdrant::quantization_config::Quantization::Binary(binary)) => { - assert_eq!(binary.always_ram, Some(false)); - } - (1, qdrant::quantization_config::Quantization::Scalar(scalar)) => { - assert_eq!(scalar.r#type, QuantizationType::Int8 as i32); - assert_eq!(scalar.quantile, Some(0.99)); - assert_eq!(scalar.always_ram, Some(false)); - } - (2, qdrant::quantization_config::Quantization::Product(product)) => { - assert_eq!(product.compression, CompressionRatio::X16 as i32); - assert_eq!(product.always_ram, Some(false)); - } - _ => panic!("unexpected quantization"), + #[expect( + deprecated, + reason = "the test verifies Qdrant's legacy always_ram quantization contract" + )] + match (quantization, quantization_config) { + (Quantization::Binary, qdrant::quantization_config::Quantization::Binary(binary)) => { + assert_eq!(binary.always_ram, Some(false)); } - assert!(state.index_creations >= 1); - assert_eq!(state.field_indexes[0].collection_name, "semantic"); - assert_eq!(state.field_indexes[0].field_name, "litellm_cache_key"); - assert_eq!( - state.field_indexes[0].field_type, - Some(qdrant::FieldType::Keyword as i32) - ); - server.stop(); + (Quantization::Scalar, qdrant::quantization_config::Quantization::Scalar(scalar)) => { + assert_eq!(scalar.r#type, QuantizationType::Int8 as i32); + assert_eq!(scalar.quantile, Some(0.99)); + assert_eq!(scalar.always_ram, Some(false)); + } + (Quantization::Product, qdrant::quantization_config::Quantization::Product(product)) => { + assert_eq!(product.compression, CompressionRatio::X16 as i32); + assert_eq!(product.always_ram, Some(false)); + } + _ => panic!("unexpected quantization"), } + assert!(state.index_creations >= 1); + assert_eq!(state.field_indexes[0].collection_name, "semantic"); + assert_eq!(state.field_indexes[0].field_name, "litellm_cache_key"); + assert_eq!( + state.field_indexes[0].field_type, + Some(qdrant::FieldType::Keyword as i32) + ); + server.stop(); } +#[rstest] #[tokio::test(flavor = "multi_thread")] async fn existing_collection_skips_create_and_index_failure_is_non_fatal() { let server = FakeQdrant::start(FakeState { @@ -160,19 +166,25 @@ async fn existing_collection_skips_create_and_index_failure_is_non_fatal() { ..Default::default() }) .await; - let _cache = connect(&server, [("hello", vec![1.0, 0.0])]).await; + let cache = connect(&server, [("hello", vec![1.0, 0.0])]).await; + assert_eq!(cache.collection_name(), "semantic"); + assert_eq!(cache.similarity_threshold(), 0.9); + assert_eq!(cache.vector_size(), 2); let state = server.state.lock().unwrap(); assert!(state.created_collections.is_empty()); assert!(state.index_creations >= 1); server.stop(); } +#[rstest] #[tokio::test(flavor = "multi_thread")] -async fn async_and_sync_set_get_store_exact_payload() { +async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) { let server = FakeQdrant::start(FakeState::default()).await; let cache = Arc::new(connect(&server, [("hello", vec![1.0, 0.0])]).await); - let ctx = context("hello"); - let entry = value(json!({"answer": 42})); + let ctx = SemanticCacheContext { + metadata: Some(json!({"tenant": "team"})), + ..context("hello") + }; cache .async_set_cache("key", entry.clone(), ctx.clone()) .await @@ -188,10 +200,8 @@ async fn async_and_sync_set_get_store_exact_payload() { payload_keys.sort(); assert_eq!(payload_keys, ["litellm_cache_key", "response", "text"]); assert_eq!(payload["litellm_cache_key"], Value::from("key")); - assert_eq!( - payload["response"], - Value::from(String::from_utf8(ResponseCacheCodec.encode(&entry).unwrap()).unwrap()) - ); + assert_eq!(payload["text"], Value::from("hello")); + assert_eq!(payload["response"], Value::from(entry.to_string())); } let sync_entry = entry.clone(); let sync_cache = cache.clone(); @@ -207,208 +217,276 @@ async fn async_and_sync_set_get_store_exact_payload() { }) .await .unwrap(); + assert_eq!( + *cache.embedder().calls.lock().unwrap(), + vec![("hello".to_owned(), ctx.metadata.clone()); 4] + ); server.stop(); } +#[rstest] +#[case::content_parts_skip_images( + json!([ + {"role": "user", "content": "hello"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "world"}, + {"type": "image_url", "image_url": {"url": "ignored"}}, + {"type": "text", "text": "!"}, + ], + }, + ]), + "helloworld!" +)] +#[case::search_results_and_compact_citations( + json!([{ + "role": "tool", + "content": null, + "search_results": [{ + "source": "source", + "title": "title", + "content": [{"text": "body"}], + "citations": {"page": 1, "section": "intro"}, + }], + }]), + r#"sourcetitlebody{"page":1,"section":"intro"}"# +)] #[tokio::test(flavor = "multi_thread")] -async fn misses_and_payload_validation_are_safe() { +async fn prompt_matches_python_message_rules( + #[case] messages: JsonValue, + #[case] prompt: &'static str, + entry: JsonValue, +) { + let server = FakeQdrant::start(FakeState::default()).await; + let cache = connect(&server, [(prompt, vec![1.0, 0.0])]).await; + let context = SemanticCacheContext { + messages: Some(messages), + ..Default::default() + }; + + cache.async_set_cache("key", entry, context).await.unwrap(); + + assert_eq!(cache.embedder().calls.lock().unwrap()[0].0, prompt); + assert_eq!( + server.state.lock().unwrap().points[0].payload["text"], + Value::from(prompt) + ); + server.stop(); +} + +#[rstest] +#[case::no_messages(SemanticCacheContext::default())] +#[case::empty_messages(SemanticCacheContext { messages: Some(json!([])), ..Default::default() })] +#[case::responses_input_is_not_read(SemanticCacheContext { input: Some(json!("hello")), ..Default::default() })] +#[tokio::test(flavor = "multi_thread")] +async fn requests_without_messages_are_missing_a_prompt( + #[case] context: SemanticCacheContext, + entry: JsonValue, +) { + let server = FakeQdrant::start(FakeState::default()).await; + let cache = connect(&server, [("hello", vec![1.0, 0.0])]).await; + + assert_eq!( + cache.async_set_cache("key", entry, context.clone()).await, + Err(Error::MissingPrompt) + ); + assert_eq!( + cache.async_get_cache("key", &context).await, + Err(Error::MissingPrompt) + ); + assert!(cache.embedder().calls.lock().unwrap().is_empty()); + server.stop(); +} + +#[rstest] +#[case::other_key("other", "hello", None)] +#[case::below_similarity_threshold("key", "near", None)] +#[tokio::test(flavor = "multi_thread")] +async fn misses_and_payload_validation_are_safe( + #[case] key: &str, + #[case] prompt: &str, + #[case] numeric_key_point: Option, + entry: JsonValue, +) { let server = FakeQdrant::start(FakeState::default()).await; let cache = connect( &server, [("hello", vec![1.0, 0.0]), ("near", vec![0.7, 0.71414286])], ) .await; - let entry = value(json!({"answer": 1})); cache .async_set_cache("key", entry, context("hello")) .await .unwrap(); + if let Some(id) = numeric_key_point { + server.insert_point(StoredPoint { + id: Some(PointId::from(id)), + vector: vec![1.0, 0.0], + payload: Payload::try_from(json!({ + "litellm_cache_key": id, + "response": "{}", + })) + .unwrap() + .into(), + }); + } + assert_eq!( - cache - .async_get_cache("other", &context("hello")) - .await - .unwrap(), - None - ); - assert_eq!( - cache - .async_get_cache("key", &context("near")) - .await - .unwrap(), - None - ); - server.insert_point(StoredPoint { - id: Some(PointId::from(99_u64)), - vector: vec![1.0, 0.0], - payload: Payload::try_from(json!({ - "litellm_cache_key": 99, - "response": "{}", - })) - .unwrap() - .into(), - }); - assert_eq!( - cache - .async_get_cache("99", &context("hello")) - .await - .unwrap(), + cache.async_get_cache(key, &context(prompt)).await.unwrap(), None ); server.stop(); } +#[rstest] +#[case::hit("key", context("hello"), Ok((true, Some(1.0))))] +#[case::below_similarity_threshold("key", context("near"), Ok((false, Some(0.7))))] +#[case::no_results("other", context("hello"), Ok((false, Some(0.0))))] +#[case::no_prompt("key", SemanticCacheContext::default(), Err(Error::MissingPrompt))] #[tokio::test(flavor = "multi_thread")] -async fn decoding_errors_missing_prompt_pipeline_and_ttl_behave_as_required() { +async fn lookup_reports_python_semantic_similarity( + #[case] key: &'static str, + #[case] context: SemanticCacheContext, + #[case] expected: Result<(bool, Option), Error>, + #[values(false, true)] use_async: bool, + entry: JsonValue, +) { let server = FakeQdrant::start(FakeState::default()).await; - let cache = connect(&server, [("one", vec![1.0, 0.0]), ("two", vec![0.0, 1.0])]).await; - let empty = SemanticCacheContext::default(); - assert_eq!( - cache - .async_set_cache("key", value(json!({})), empty.clone()) - .await, - Err(Error::MissingPrompt) + let cache = Arc::new( + connect( + &server, + [("hello", vec![1.0, 0.0]), ("near", vec![0.7, 0.71414286])], + ) + .await, ); + cache + .async_set_cache("key", entry.clone(), self::context("hello")) + .await + .unwrap(); + server.insert_point(StoredPoint { + id: Some(PointId::from(99_u64)), + vector: vec![1.0, 0.0], + payload: Payload::try_from(json!({"litellm_cache_key": 99, "response": "{}"})) + .unwrap() + .into(), + }); + + let lookup = if use_async { + cache.async_get_cache_with_similarity(key, &context).await + } else { + let cache = Arc::clone(&cache); + tokio::task::spawn_blocking(move || cache.get_cache_with_similarity(key, &context)) + .await + .unwrap() + }; + + match (lookup, expected) { + (Ok(SemanticLookup { value, similarity }), Ok((hit, expected))) => { + assert_eq!(value, hit.then_some(entry)); + assert_eq!(similarity.is_some(), expected.is_some()); + if let (Some(similarity), Some(expected)) = (similarity, expected) { + assert!((similarity - expected).abs() < 1e-6, "{similarity}"); + } + } + (lookup, expected) => assert_eq!(lookup.map(|_| ()), expected.map(|_| ())), + } + server.stop(); +} + +#[rstest] +#[case::codec_decodes_the_payload(Some(json!("{\"a\":1}")), Ok(Some(json!({"a": 1}))))] +#[case::undecodable_response(Some(json!("not json")), Err(Error::InvalidEntry))] +#[case::non_string_response(Some(json!(1)), Err(Error::InvalidEntry))] +#[case::missing_response(None, Err(Error::InvalidEntry))] +#[tokio::test(flavor = "multi_thread")] +async fn stored_responses_go_through_the_codec( + #[case] response: Option, + #[case] expected: Result, Error>, +) { + let server = FakeQdrant::start(FakeState::default()).await; + let cache = connect(&server, [("hello", vec![1.0, 0.0])]).await; + let mut payload = serde_json::Map::new(); + payload.insert("litellm_cache_key".to_owned(), json!("key")); + if let Some(response) = response { + payload.insert("response".to_owned(), response); + } + server.insert_point(StoredPoint { + id: Some(PointId::from(1_u64)), + vector: vec![1.0, 0.0], + payload: Payload::try_from(JsonValue::Object(payload)) + .unwrap() + .into(), + }); + assert_eq!( - cache.async_get_cache("key", &empty).await, - Err(Error::MissingPrompt) + cache.async_get_cache("key", &context("hello")).await, + expected ); + server.stop(); +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn embedding_failures_propagate() { + let server = FakeQdrant::start(FakeState::default()).await; + let cache = connect(&server, []).await; + assert_eq!( cache.async_get_cache("key", &context("unknown")).await, Err(Error::Unavailable) ); + server.stop(); +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn ttl_is_ignored_and_entries_do_not_expire(entry: JsonValue) { + let server = FakeQdrant::start(FakeState::default()).await; + let cache = connect(&server, [("one", vec![1.0, 0.0])]).await; + let ctx = context("one").with_ttl(Some(Duration::from_secs(1))); + + assert_eq!(cache.get_ttl(&ctx), None); cache - .async_set_cache( - "ttl", - value(json!({"ttl": true})), - context("one").with_ttl(Some(Duration::from_secs(1))), - ) + .async_set_cache("ttl", entry, ctx.clone()) .await .unwrap(); tokio::time::sleep(Duration::from_millis(1_100)).await; - assert!( - cache - .async_get_cache( - "ttl", - &context("one").with_ttl(Some(Duration::from_secs(1))), - ) - .await - .unwrap() - .is_some() - ); + assert!(cache.async_get_cache("ttl", &ctx).await.unwrap().is_some()); + server.stop(); +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn pipeline_upserts_each_entry_and_waits_for_indexing() { + let server = FakeQdrant::start(FakeState::default()).await; + let cache = connect(&server, [("one", vec![1.0, 0.0])]).await; + cache .async_set_cache_pipeline( vec![ - ("one".to_owned(), value(json!({"n": 1}))), - ("two".to_owned(), value(json!({"n": 2}))), + ("one".to_owned(), json!({"n": 1})), + ("two".to_owned(), json!({"n": 2})), ], context("one"), ) .await .unwrap(); - assert!( - cache - .async_get_cache("one", &context("one")) - .await - .unwrap() - .is_some() - ); - assert!( - cache - .async_get_cache("two", &context("one")) - .await - .unwrap() - .is_some() - ); + + for (key, value) in [("one", json!({"n": 1})), ("two", json!({"n": 2}))] { + assert_eq!( + cache.async_get_cache(key, &context("one")).await.unwrap(), + Some(value) + ); + } assert_eq!( server.state.lock().unwrap().upsert_waits, - vec![Some(true), Some(true), Some(true)] - ); - assert_eq!(cache.get_ttl(&context("one")), None); - assert_eq!( - cache.test_connection().await, - Err(Error::UnsupportedOperation) - ); - server.stop(); -} - -#[tokio::test(flavor = "multi_thread")] -async fn response_payloads_decode_and_invalid_entries_fail() { - let server = FakeQdrant::start(FakeState::default()).await; - let cache = connect(&server, [("hello", vec![1.0, 0.0])]).await; - for (key, response) in [ - ("python", json!("{'timestamp': 1.0, 'response': {'a': 1}}")), - ("garbage", json!("not json")), - ("missing", json!("unused")), - ] { - let mut payload = serde_json::Map::new(); - payload.insert("litellm_cache_key".to_owned(), json!(key)); - if key != "missing" { - payload.insert("response".to_owned(), response); - } - server.insert_point(StoredPoint { - id: Some(PointId::from(key.len() as u64)), - vector: vec![1.0, 0.0], - payload: Payload::try_from(JsonValue::Object(payload)) - .unwrap() - .into(), - }); - } - assert_eq!( - cache - .async_get_cache("python", &context("hello")) - .await - .unwrap(), - Some(value(json!({"a": 1}))) - ); - assert_eq!( - cache.async_get_cache("garbage", &context("hello")).await, - Err(Error::InvalidEntry) - ); - assert_eq!( - cache.async_get_cache("missing", &context("hello")).await, - Err(Error::InvalidEntry) - ); - server.stop(); -} - -#[tokio::test(flavor = "multi_thread")] -async fn response_cache_facade_turns_invalid_entry_into_miss() { - let server = FakeQdrant::start(FakeState::default()).await; - let cache = Arc::new(connect(&server, [("hello", vec![1.0, 0.0])]).await); - let request = ResponseCacheRequest::::new(CacheKeyInput { - preset: Some("key".to_owned()), - ..Default::default() - }) - .with_context(context("hello")); - let response = json!({"answer": 42}); - let facade = ResponseCache::new(cache.clone()); - facade - .async_store(&request, response.clone(), Duration::from_secs(1)) - .await - .unwrap(); - assert_eq!( - facade - .async_lookup(&request, Duration::from_secs(1)) - .await - .unwrap(), - Some(response) - ); - { - let mut state = server.state.lock().unwrap(); - state.points[0] - .payload - .insert("response".to_owned(), Value::from("not json")); - } - assert_eq!( - facade - .async_lookup(&request, Duration::from_secs(1)) - .await - .unwrap(), - None + vec![Some(true), Some(true)] ); server.stop(); } +#[rstest] #[tokio::test(flavor = "multi_thread")] async fn stopped_qdrant_server_maps_to_unavailable() { let server = FakeQdrant::start(FakeState::default()).await; @@ -420,3 +498,28 @@ async fn stopped_qdrant_server_maps_to_unavailable() { Err(Error::Unavailable) ); } + +/// `_payload_matches_cache_key` compares `str(cached_key) == str(key)`, so a point whose stored +/// key is the number 99 answers a lookup for `"99"`. +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn numeric_stored_cache_keys_match_like_python_str() { + let server = FakeQdrant::start(FakeState::default()).await; + let cache = connect(&server, [("hello", vec![1.0, 0.0])]).await; + server.insert_point(StoredPoint { + id: Some(PointId::from(99_u64)), + vector: vec![1.0, 0.0], + payload: Payload::try_from(json!({"litellm_cache_key": 99, "response": "{}"})) + .unwrap() + .into(), + }); + + let lookup = cache + .async_get_cache_with_similarity("99", &context("hello")) + .await + .unwrap(); + + assert_eq!(lookup.value, Some(json!({}))); + assert!((lookup.similarity.unwrap() - 1.0).abs() < 1e-6); + server.stop(); +} diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/support/mod.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/support/mod.rs index 9a556ae7df5..695fceeac44 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/tests/support/mod.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/support/mod.rs @@ -1,15 +1,16 @@ +#![allow(dead_code)] + use std::{ collections::{HashMap, HashSet}, net::SocketAddr, sync::{Arc, Mutex}, }; -use qdrant_client::qdrant::collections_server::CollectionsServer; use qdrant_client::qdrant::{ self, CollectionExists, CollectionExistsRequest, CollectionExistsResponse, CollectionOperationResponse, CreateCollection, CreateFieldIndexCollection, Filter, PointId, PointsOperationResponse, ScoredPoint, SearchPoints, SearchResponse, Value, Vector, Vectors, - collections_server::Collections, + collections_server::{Collections, CollectionsServer}, points_server::{Points, PointsServer}, }; use tokio::sync::oneshot; diff --git a/litellm-rust/crates/cache-redis-semantic/Cargo.toml b/litellm-rust/crates/cache-redis-semantic/Cargo.toml index 9a8755a189e..fe5a317cec0 100644 --- a/litellm-rust/crates/cache-redis-semantic/Cargo.toml +++ b/litellm-rust/crates/cache-redis-semantic/Cargo.toml @@ -8,14 +8,12 @@ repository.workspace = true [dependencies] litellm-cache.workspace = true litellm-cache-redis.workspace = true -litellm-cache-response.workspace = true redis = { version = "1.7.0", features = ["tls-rustls"] } -r2d2 = "0.8.10" -serde_json.workspace = true sha2.workspace = true -tokio.workspace = true [dev-dependencies] +litellm-cache-testing.workspace = true redis-test = "1.0.4" +rstest.workspace = true serde_json.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/cache-redis-semantic/src/cache.rs b/litellm-rust/crates/cache-redis-semantic/src/cache.rs index e0ac31f3630..02feb3cbdf4 100644 --- a/litellm-rust/crates/cache-redis-semantic/src/cache.rs +++ b/litellm-rust/crates/cache-redis-semantic/src/cache.rs @@ -1,105 +1,36 @@ use std::{ - future::Future, - sync::{Arc, OnceLock}, + sync::Arc, time::{Duration, SystemTime, UNIX_EPOCH}, }; use litellm_cache::{ - BaseCache, CacheCodec, CacheConnectionResult, CacheConnectionStatus, Error, - SemanticCacheContext, + BaseCache, CacheCodec, Error, SemanticCacheContext, + semantic::{Embedder, SemanticCache, SemanticLookup, prompt_from_context}, }; use litellm_cache_redis::{ RedisTopology, connection::{ConnectionRef, Connections}, }; -use litellm_cache_response::{CacheEntry, ResponseCacheCodec}; -use serde_json::Value; use sha2::{Digest, Sha256}; -use crate::prompt::prompt_from_context; - -const CACHE_KEY_FIELD: &str = "litellm_cache_key"; -const VECTOR_FIELD: &str = "prompt_vector"; - -pub trait Embedder: Send + Sync + 'static { - fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error>; - - fn async_embed( - &self, - prompt: &str, - metadata: Option<&Value>, - ) -> impl Future, Error>> + Send; -} - -#[derive(Clone, Debug)] -pub struct RedisSemanticConfig { - pub index_name: String, - pub similarity_threshold: f32, -} +use crate::{ + RedisSemanticConfig, + index::{CACHE_KEY_FIELD, Index, VECTOR_FIELD}, + reply::{bytes_field, first_document, number_field, string_field}, +}; struct Inner { - index_name: String, + index: Index, distance_threshold: f64, - resolved_index: OnceLock, - codec: ResponseCacheCodec, clock: fn() -> f64, } impl Inner { - fn new(config: RedisSemanticConfig) -> Self { + fn new(config: RedisSemanticConfig, clock: fn() -> f64) -> Self { Self { - index_name: config.index_name, + index: Index::new(config.index_name), distance_threshold: 1.0 - f64::from(config.similarity_threshold), - resolved_index: OnceLock::new(), - codec: ResponseCacheCodec, - clock: timestamp, - } - } - - fn ensure_index( - &self, - connection: &mut ConnectionRef<'_>, - dims: usize, - ) -> Result { - if let Some(name) = self.resolved_index.get() { - return Ok(name.clone()); - } - let name = match index_compatible(connection, &self.index_name, dims)? { - Some(true) => self.index_name.clone(), - Some(false) => self.isolated_index(connection, dims)?, - None => match create_index(connection, &self.index_name, dims) { - Ok(()) => self.index_name.clone(), - Err(_) => match index_compatible(connection, &self.index_name, dims)? { - Some(true) => self.index_name.clone(), - Some(false) => self.isolated_index(connection, dims)?, - None => return Err(Error::Unavailable), - }, - }, - }; - let _ = self.resolved_index.set(name.clone()); - Ok(name) - } - - fn isolated_index( - &self, - connection: &mut ConnectionRef<'_>, - dims: usize, - ) -> Result { - let name = format!("{}_isolated", self.index_name); - match index_compatible(connection, &name, dims)? { - Some(true) => Ok(name), - Some(false) => { - redis::cmd("FT.DROPINDEX") - .arg(&name) - .query::<()>(connection) - .map_err(|_| Error::Unavailable)?; - create_index(connection, &name, dims)?; - Ok(name) - } - None => { - create_index(connection, &name, dims)?; - Ok(name) - } + clock, } } @@ -107,15 +38,14 @@ impl Inner { &self, connection: &mut ConnectionRef<'_>, tag: &str, - value: &CacheEntry, + response: Vec, prompt: &str, vector: &[f32], ttl: Option, ) -> Result<(), Error> { - let index = self.ensure_index(connection, vector.len())?; + let index = self.index.ensure(connection, vector.len())?; let entry_id = entry_id(prompt, tag); let hash_key = format!("{index}:{entry_id}"); - let response = self.codec.encode(value)?; redis::cmd("HSET") .arg(&hash_key) .arg("entry_id") @@ -149,8 +79,8 @@ impl Inner { connection: &mut ConnectionRef<'_>, tag: &str, vector: &[f32], - ) -> Result, Error> { - let index = self.ensure_index(connection, vector.len())?; + ) -> Result>, Error> { + let index = self.index.ensure(connection, vector.len())?; let query = format!( "(@{CACHE_KEY_FIELD}:{{{}}})=>[KNN 1 @{VECTOR_FIELD} $vector AS vector_distance]", escape_tag(tag) @@ -183,57 +113,80 @@ impl Inner { .query::(connection) .map_err(|_| Error::Unavailable)?; let Some(fields) = first_document(&result) else { - return Ok(None); + return Ok(SemanticLookup::miss(Some(0.0))); }; if string_field(fields, CACHE_KEY_FIELD).as_deref() != Some(tag) { - return Ok(None); + return Ok(SemanticLookup::miss(Some(0.0))); } - if number_field(fields, "vector_distance") - .is_none_or(|distance| distance > self.distance_threshold) - { - return Ok(None); - } - let Some(response) = bytes_field(fields, "response") else { - return Ok(None); + // redisvl's range query only returns entries within the distance threshold, so a + // farther hit reads as no result. + let Some(distance) = number_field(fields, "vector_distance") + .filter(|distance| *distance <= self.distance_threshold) + else { + return Ok(SemanticLookup::miss(Some(0.0))); }; - self.codec.decode(&response).map(Some) - } -} - -pub struct RedisSemanticCache { - connections: Arc>, - embedder: E, - inner: Arc, -} - -impl RedisSemanticCache { - pub fn new(url: &str, embedder: E, config: RedisSemanticConfig) -> Result { - Ok(Self { - connections: Arc::new(Connections::open(url, &RedisTopology::Standalone)?), - embedder, - inner: Arc::new(Inner::new(config)), + let Some(response) = bytes_field(fields, "response") else { + return Ok(SemanticLookup::miss(Some(0.0))); + }; + Ok(SemanticLookup { + value: Some(response), + similarity: Some(1.0 - distance), }) } } -impl RedisSemanticCache { - pub fn with_connection(connection: C, embedder: E, config: RedisSemanticConfig) -> Self { +/// `RedisSemanticCache`: a redisvl-compatible semantic index on Redis Stack. Values go through +/// the injected codec, so the response layer decides what a cached entry is. +pub struct RedisSemanticCache { + connections: Arc>, + embedder: E, + codec: S, + inner: Arc, +} + +impl RedisSemanticCache { + pub fn new( + url: &str, + embedder: E, + codec: S, + config: RedisSemanticConfig, + ) -> Result { + Ok(Self { + connections: Arc::new(Connections::open(url, &RedisTopology::Standalone)?), + embedder, + codec, + inner: Arc::new(Inner::new(config, timestamp)), + }) + } +} + +impl RedisSemanticCache +where + E: Embedder, + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + pub fn with_connection( + connection: C, + embedder: E, + codec: S, + config: RedisSemanticConfig, + ) -> Self { Self { connections: Arc::new(Connections::fixed(connection)), embedder, - inner: Arc::new(Inner::new(config)), + codec, + inner: Arc::new(Inner::new(config, timestamp)), } } pub fn with_clock(self, clock: fn() -> f64) -> Self { + let config = RedisSemanticConfig { + index_name: self.index_name().to_owned(), + similarity_threshold: self.similarity_threshold(), + }; Self { - inner: Arc::new(Inner { - index_name: self.inner.index_name.clone(), - distance_threshold: self.inner.distance_threshold, - resolved_index: OnceLock::new(), - codec: self.inner.codec, - clock, - }), + inner: Arc::new(Inner::new(config, clock)), ..self } } @@ -243,7 +196,7 @@ impl RedisSemanticCache< } pub fn index_name(&self) -> &str { - &self.inner.index_name + self.inner.index.name() } pub fn similarity_threshold(&self) -> f32 { @@ -253,12 +206,25 @@ impl RedisSemanticCache< fn tag<'a>(key: &'a str, context: &'a SemanticCacheContext) -> &'a str { context.scope.as_deref().unwrap_or(key) } + + fn decode(&self, lookup: SemanticLookup>) -> Result, Error> { + Ok(SemanticLookup { + value: lookup + .value + .map(|bytes| self.codec.decode(&bytes)) + .transpose()?, + similarity: lookup.similarity, + }) + } } -impl BaseCache - for RedisSemanticCache +impl BaseCache for RedisSemanticCache +where + E: Embedder, + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, { - type Value = CacheEntry; + type Value = S::Value; type Context = SemanticCacheContext; fn get_ttl(&self, context: &Self::Context) -> Option { @@ -274,22 +240,18 @@ impl BaseCache let Some(prompt) = prompt_from_context(context) else { return Ok(()); }; + let response = self.codec.encode(&value)?; let vector = self.embedder.embed(&prompt, context.metadata.as_ref())?; - let tag = Self::tag(key, context).to_string(); + let tag = Self::tag(key, context); self.connections.execute(|connection| { self.inner - .store(connection, &tag, &value, &prompt, &vector, context.ttl) + .store(connection, tag, response, &prompt, &vector, context.ttl) }) } fn get_cache(&self, key: &str, context: &Self::Context) -> Result, Error> { - let Some(prompt) = prompt_from_context(context) else { - return Ok(None); - }; - let vector = self.embedder.embed(&prompt, context.metadata.as_ref())?; - let tag = Self::tag(key, context).to_string(); - self.connections - .execute(|connection| self.inner.lookup(connection, &tag, &vector)) + self.get_cache_with_similarity(key, context) + .map(|lookup| lookup.value) } async fn async_set_cache( @@ -301,14 +263,15 @@ impl BaseCache let Some(prompt) = prompt_from_context(&context) else { return Ok(()); }; + let response = self.codec.encode(&value)?; let vector = self .embedder .async_embed(&prompt, context.metadata.as_ref()) .await?; - let tag = Self::tag(key, &context).to_string(); + let tag = Self::tag(key, &context).to_owned(); let inner = Arc::clone(&self.inner); Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - inner.store(connection, &tag, &value, &prompt, &vector, context.ttl) + inner.store(connection, &tag, response, &prompt, &vector, context.ttl) }) .await } @@ -318,49 +281,54 @@ impl BaseCache key: &str, context: &Self::Context, ) -> Result, Error> { + self.async_get_cache_with_similarity(key, context) + .await + .map(|lookup| lookup.value) + } +} + +/// Python stamps a similarity of `0.0` when there is no prompt or no hit in the key's scope. +impl SemanticCache for RedisSemanticCache +where + E: Embedder, + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + fn get_cache_with_similarity( + &self, + key: &str, + context: &Self::Context, + ) -> Result, Error> { let Some(prompt) = prompt_from_context(context) else { - return Ok(None); + return Ok(SemanticLookup::miss(Some(0.0))); + }; + let vector = self.embedder.embed(&prompt, context.metadata.as_ref())?; + let tag = Self::tag(key, context); + let lookup = self + .connections + .execute(|connection| self.inner.lookup(connection, tag, &vector))?; + self.decode(lookup) + } + + async fn async_get_cache_with_similarity( + &self, + key: &str, + context: &Self::Context, + ) -> Result, Error> { + let Some(prompt) = prompt_from_context(context) else { + return Ok(SemanticLookup::miss(Some(0.0))); }; let vector = self .embedder .async_embed(&prompt, context.metadata.as_ref()) .await?; - let tag = Self::tag(key, context).to_string(); + let tag = Self::tag(key, context).to_owned(); let inner = Arc::clone(&self.inner); - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { + let lookup = Connections::run_blocking(Arc::clone(&self.connections), move |connection| { inner.lookup(connection, &tag, &vector) }) - .await - } - - async fn disconnect(&self) -> Result<(), Error> { - Ok(()) - } - - async fn test_connection(&self) -> Result { - match Connections::run_blocking(Arc::clone(&self.connections), |connection| { - Ok(match redis::cmd("PING").query::(connection) { - Ok(_) => CacheConnectionResult { - status: CacheConnectionStatus::Success, - message: "Redis cache connection test successful".into(), - error: None, - }, - Err(error) => CacheConnectionResult { - status: CacheConnectionStatus::Failed, - message: format!("Redis connection failed: {error}"), - error: Some(error.to_string()), - }, - }) - }) - .await - { - Ok(result) => Ok(result), - Err(error) => Ok(CacheConnectionResult { - status: CacheConnectionStatus::Failed, - message: format!("Redis connection failed: {error}"), - error: Some(error.to_string()), - }), - } + .await?; + self.decode(lookup) } } @@ -387,228 +355,46 @@ fn vector_buffer(vector: &[f32]) -> Vec { } fn escape_tag(value: &str) -> String { - value - .chars() - .flat_map(|ch| { - if matches!( - ch, - ',' | '.' - | '<' - | '>' - | '{' - | '}' - | '[' - | ']' - | '\\' - | '"' - | '\'' - | ':' - | ';' - | '!' - | '@' - | '#' - | '$' - | '%' - | '^' - | '&' - | '*' - | '(' - | ')' - | '-' - | '+' - | '=' - | '~' - | '|' - | '/' - | ' ' - | '?' - ) { - vec!['\\', ch] - } else { - vec![ch] - } - }) - .collect() -} - -fn create_index(connection: &mut ConnectionRef<'_>, name: &str, dims: usize) -> Result<(), Error> { - redis::cmd("FT.CREATE") - .arg(name) - .arg("ON") - .arg("HASH") - .arg("PREFIX") - .arg(1) - .arg(name) - .arg("SCORE") - .arg(1.0) - .arg("SCHEMA") - .arg("prompt") - .arg("TEXT") - .arg("WEIGHT") - .arg(1) - .arg("response") - .arg("TEXT") - .arg("WEIGHT") - .arg(1) - .arg("inserted_at") - .arg("NUMERIC") - .arg("updated_at") - .arg("NUMERIC") - .arg(VECTOR_FIELD) - .arg("VECTOR") - .arg("FLAT") - .arg(6) - .arg("TYPE") - .arg("FLOAT32") - .arg("DIM") - .arg(dims) - .arg("DISTANCE_METRIC") - .arg("COSINE") - .arg(CACHE_KEY_FIELD) - .arg("TAG") - .arg("SEPARATOR") - .arg(",") - .query::<()>(connection) - .map_err(|_| Error::Unavailable) -} - -fn index_compatible( - connection: &mut ConnectionRef<'_>, - name: &str, - dims: usize, -) -> Result, Error> { - let info = match redis::cmd("FT.INFO") - .arg(name) - .query::(connection) - { - Ok(info) => info, - Err(error) if unknown_index(&error) => return Ok(None), - Err(_) => return Err(Error::Unavailable), - }; - Ok(Some(schema_compatible(&info, dims))) -} - -fn unknown_index(error: &redis::RedisError) -> bool { - let message = error.to_string().to_lowercase(); - message.contains("unknown") && message.contains("index") -} - -fn schema_compatible(info: &redis::Value, dims: usize) -> bool { - let redis::Value::Array(entries) = info else { - return false; - }; - let attributes = entries - .as_chunks::<2>() - .0 - .iter() - .find(|pair| string_value(&pair[0]).as_deref() == Some("attributes")) - .map(|pair| &pair[1]); - let Some(redis::Value::Array(attributes)) = attributes else { - return false; - }; - let fields = attributes - .iter() - .map(|attribute| { - let redis::Value::Array(attribute) = attribute else { - return (None, None, None, None, None); - }; - let mut name = None; - let mut field_type = None; - let mut dim = None; - let mut data_type = None; - let mut distance_metric = None; - for pair in attribute.as_chunks::<2>().0 { - match string_value(&pair[0]).as_deref() { - Some("identifier") => name = string_value(&pair[1]), - Some("type") => field_type = string_value(&pair[1]), - Some("dim") => dim = number_value(&pair[1]), - Some("data_type") => data_type = string_value(&pair[1]), - Some("distance_metric") => distance_metric = string_value(&pair[1]), - _ => {} - } - } - (name, field_type, dim, data_type, distance_metric) - }) - .collect::>(); - let has_field = |name: &str, field_type: &str| { - fields - .iter() - .any(|(n, t, ..)| n.as_deref() == Some(name) && t.as_deref() == Some(field_type)) - }; - has_field("prompt", "TEXT") - && has_field("response", "TEXT") - && has_field("inserted_at", "NUMERIC") - && has_field("updated_at", "NUMERIC") - && has_field(CACHE_KEY_FIELD, "TAG") - && fields.iter().any(|(n, t, d, data, metric)| { - n.as_deref() == Some(VECTOR_FIELD) - && t.as_deref() == Some("VECTOR") - && *d == Some(dims as f64) - && data - .as_deref() - .is_some_and(|data| data.eq_ignore_ascii_case("float32")) - && metric - .as_deref() - .is_some_and(|metric| metric.eq_ignore_ascii_case("cosine")) - }) -} - -fn string_value(value: &redis::Value) -> Option { - match value { - redis::Value::BulkString(bytes) => String::from_utf8(bytes.clone()).ok(), - redis::Value::SimpleString(text) => Some(text.clone()), - redis::Value::VerbatimString { text, .. } => Some(text.clone()), - _ => None, - } -} - -fn number_value(value: &redis::Value) -> Option { - match value { - redis::Value::Int(number) => Some(*number as f64), - redis::Value::Double(number) => Some(*number), - _ => string_value(value).and_then(|text| text.parse().ok()), - } -} - -fn first_document(result: &redis::Value) -> Option<&[redis::Value]> { - let redis::Value::Array(items) = result else { - return None; - }; - let [count, _document_id, fields, ..] = items.as_slice() else { - return None; - }; - if !matches!(count, redis::Value::Int(count) if *count > 0) { - return None; - } - match fields { - redis::Value::Array(fields) => Some(fields.as_slice()), - _ => None, - } -} - -fn field_value<'a>(fields: &'a [redis::Value], name: &str) -> Option<&'a redis::Value> { - fields - .as_chunks::<2>() - .0 - .iter() - .find(|pair| string_value(&pair[0]).as_deref() == Some(name)) - .map(|pair| &pair[1]) -} - -fn string_field(fields: &[redis::Value], name: &str) -> Option { - field_value(fields, name).and_then(string_value) -} - -fn number_field(fields: &[redis::Value], name: &str) -> Option { - field_value(fields, name).and_then(number_value) -} - -fn bytes_field(fields: &[redis::Value], name: &str) -> Option> { - match field_value(fields, name)? { - redis::Value::BulkString(bytes) => Some(bytes.clone()), - redis::Value::SimpleString(text) => Some(text.clone().into_bytes()), - _ => None, + let mut escaped = String::with_capacity(value.len()); + for ch in value.chars() { + if matches!( + ch, + ',' | '.' + | '<' + | '>' + | '{' + | '}' + | '[' + | ']' + | '\\' + | '"' + | '\'' + | ':' + | ';' + | '!' + | '@' + | '#' + | '$' + | '%' + | '^' + | '&' + | '*' + | '(' + | ')' + | '-' + | '+' + | '=' + | '~' + | '|' + | '/' + | ' ' + | '?' + ) { + escaped.push('\\'); + } + escaped.push(ch); } + escaped } fn ttl_seconds(ttl: Duration) -> u64 { diff --git a/litellm-rust/crates/cache-redis-semantic/src/config.rs b/litellm-rust/crates/cache-redis-semantic/src/config.rs new file mode 100644 index 00000000000..6b810628be7 --- /dev/null +++ b/litellm-rust/crates/cache-redis-semantic/src/config.rs @@ -0,0 +1,8 @@ +/// `RedisSemanticCache.DEFAULT_REDIS_INDEX_NAME`. +pub const DEFAULT_INDEX_NAME: &str = "litellm_semantic_cache_index"; + +#[derive(Clone, Debug)] +pub struct RedisSemanticConfig { + pub index_name: String, + pub similarity_threshold: f32, +} diff --git a/litellm-rust/crates/cache-redis-semantic/src/index.rs b/litellm-rust/crates/cache-redis-semantic/src/index.rs new file mode 100644 index 00000000000..c141e47003a --- /dev/null +++ b/litellm-rust/crates/cache-redis-semantic/src/index.rs @@ -0,0 +1,205 @@ +use std::sync::OnceLock; + +use litellm_cache::Error; +use litellm_cache_redis::connection::ConnectionRef; + +use crate::reply::{number_value, string_value}; + +pub(crate) const CACHE_KEY_FIELD: &str = "litellm_cache_key"; +pub(crate) const VECTOR_FIELD: &str = "prompt_vector"; + +/// The redisvl `SemanticCache` index, resolved once per cache: the configured name when its +/// schema fits, else `_isolated`, recreated when that one is stale too. +pub(crate) struct Index { + name: String, + resolved: OnceLock, +} + +impl Index { + pub(crate) fn new(name: String) -> Self { + Self { + name, + resolved: OnceLock::new(), + } + } + + pub(crate) fn name(&self) -> &str { + &self.name + } + + pub(crate) fn ensure( + &self, + connection: &mut ConnectionRef<'_>, + dims: usize, + ) -> Result { + if let Some(name) = self.resolved.get() { + return Ok(name.clone()); + } + let name = match index_compatible(connection, &self.name, dims)? { + Some(true) => self.name.clone(), + Some(false) => self.isolated(connection, dims)?, + None => match create_index(connection, &self.name, dims) { + Ok(()) => self.name.clone(), + Err(_) => match index_compatible(connection, &self.name, dims)? { + Some(true) => self.name.clone(), + Some(false) => self.isolated(connection, dims)?, + None => return Err(Error::Unavailable), + }, + }, + }; + let _ = self.resolved.set(name.clone()); + Ok(name) + } + + fn isolated(&self, connection: &mut ConnectionRef<'_>, dims: usize) -> Result { + let name = format!("{}_isolated", self.name); + match index_compatible(connection, &name, dims)? { + Some(true) => Ok(name), + Some(false) => { + redis::cmd("FT.DROPINDEX") + .arg(&name) + .query::<()>(connection) + .map_err(|_| Error::Unavailable)?; + create_index(connection, &name, dims)?; + Ok(name) + } + None => { + create_index(connection, &name, dims)?; + Ok(name) + } + } + } +} + +fn create_index(connection: &mut ConnectionRef<'_>, name: &str, dims: usize) -> Result<(), Error> { + redis::cmd("FT.CREATE") + .arg(name) + .arg("ON") + .arg("HASH") + .arg("PREFIX") + .arg(1) + .arg(name) + .arg("SCORE") + .arg(1.0) + .arg("SCHEMA") + .arg("prompt") + .arg("TEXT") + .arg("WEIGHT") + .arg(1) + .arg("response") + .arg("TEXT") + .arg("WEIGHT") + .arg(1) + .arg("inserted_at") + .arg("NUMERIC") + .arg("updated_at") + .arg("NUMERIC") + .arg(VECTOR_FIELD) + .arg("VECTOR") + .arg("FLAT") + .arg(6) + .arg("TYPE") + .arg("FLOAT32") + .arg("DIM") + .arg(dims) + .arg("DISTANCE_METRIC") + .arg("COSINE") + .arg(CACHE_KEY_FIELD) + .arg("TAG") + .arg("SEPARATOR") + .arg(",") + .query::<()>(connection) + .map_err(|_| Error::Unavailable) +} + +fn index_compatible( + connection: &mut ConnectionRef<'_>, + name: &str, + dims: usize, +) -> Result, Error> { + let info = match redis::cmd("FT.INFO") + .arg(name) + .query::(connection) + { + Ok(info) => info, + Err(error) if unknown_index(&error) => return Ok(None), + Err(_) => return Err(Error::Unavailable), + }; + Ok(Some(schema_compatible(&info, dims))) +} + +fn unknown_index(error: &redis::RedisError) -> bool { + let message = error.to_string().to_lowercase(); + message.contains("unknown") && message.contains("index") +} + +struct Attribute { + name: Option, + field_type: Option, + dim: Option, + data_type: Option, + distance_metric: Option, +} + +fn attribute(value: &redis::Value) -> Option { + let redis::Value::Array(pairs) = value else { + return None; + }; + let mut attribute = Attribute { + name: None, + field_type: None, + dim: None, + data_type: None, + distance_metric: None, + }; + for pair in pairs.as_chunks::<2>().0 { + match string_value(&pair[0]).as_deref() { + Some("identifier") => attribute.name = string_value(&pair[1]), + Some("type") => attribute.field_type = string_value(&pair[1]), + Some("dim") => attribute.dim = number_value(&pair[1]), + Some("data_type") => attribute.data_type = string_value(&pair[1]), + Some("distance_metric") => attribute.distance_metric = string_value(&pair[1]), + _ => {} + } + } + Some(attribute) +} + +fn schema_compatible(info: &redis::Value, dims: usize) -> bool { + let redis::Value::Array(entries) = info else { + return false; + }; + let attributes = entries + .as_chunks::<2>() + .0 + .iter() + .find(|pair| string_value(&pair[0]).as_deref() == Some("attributes")) + .map(|pair| &pair[1]); + let Some(redis::Value::Array(attributes)) = attributes else { + return false; + }; + let fields = attributes.iter().filter_map(attribute).collect::>(); + let has_field = |name: &str, field_type: &str| { + fields.iter().any(|field| { + field.name.as_deref() == Some(name) && field.field_type.as_deref() == Some(field_type) + }) + }; + has_field("prompt", "TEXT") + && has_field("response", "TEXT") + && has_field("inserted_at", "NUMERIC") + && has_field("updated_at", "NUMERIC") + && has_field(CACHE_KEY_FIELD, "TAG") + && fields.iter().any(|field| { + field.name.as_deref() == Some(VECTOR_FIELD) + && field.field_type.as_deref() == Some("VECTOR") + && field.dim == Some(dims as f64) + && field + .data_type + .as_deref() + .is_some_and(|data| data.eq_ignore_ascii_case("float32")) + && field + .distance_metric + .as_deref() + .is_some_and(|metric| metric.eq_ignore_ascii_case("cosine")) + }) +} diff --git a/litellm-rust/crates/cache-redis-semantic/src/lib.rs b/litellm-rust/crates/cache-redis-semantic/src/lib.rs index 51d0b4ba5f3..251323df6cf 100644 --- a/litellm-rust/crates/cache-redis-semantic/src/lib.rs +++ b/litellm-rust/crates/cache-redis-semantic/src/lib.rs @@ -1,5 +1,7 @@ mod cache; -mod prompt; +mod config; +mod index; +mod reply; -pub use cache::{Embedder, RedisSemanticCache, RedisSemanticConfig}; -pub use prompt::prompt_from_context; +pub use cache::RedisSemanticCache; +pub use config::{DEFAULT_INDEX_NAME, RedisSemanticConfig}; diff --git a/litellm-rust/crates/cache-redis-semantic/src/prompt.rs b/litellm-rust/crates/cache-redis-semantic/src/prompt.rs deleted file mode 100644 index b9c38e98d77..00000000000 --- a/litellm-rust/crates/cache-redis-semantic/src/prompt.rs +++ /dev/null @@ -1,97 +0,0 @@ -use litellm_cache::SemanticCacheContext; -use serde_json::Value; - -pub fn prompt_from_context(context: &SemanticCacheContext) -> Option { - if let Some(messages) = context.messages.as_ref().and_then(Value::as_array) - && !messages.is_empty() - { - return Some(messages_text(messages)); - } - let input = context.input.as_ref()?; - let mut parts = Vec::new(); - collect_input_text(input, &mut parts); - let prompt = parts.join("\n").trim().to_string(); - (!prompt.is_empty()).then_some(prompt) -} - -fn messages_text(messages: &[Value]) -> String { - let mut text = String::new(); - for message in messages { - let Some(message) = message.as_object() else { - continue; - }; - match message.get("content") { - Some(Value::String(content)) => text.push_str(content), - Some(Value::Array(parts)) => { - for part in parts { - if let Some(text_content) = part.get("text").and_then(Value::as_str) { - text.push_str(text_content); - } - } - } - _ => {} - } - text.push_str(&search_results_text(message.get("search_results"))); - } - text -} - -fn search_results_text(search_results: Option<&Value>) -> String { - let Some(Value::Array(results)) = search_results else { - return String::new(); - }; - let mut text = String::new(); - for result in results { - let Some(result) = result.as_object() else { - continue; - }; - for key in ["source", "title"] { - if let Some(value) = result.get(key).and_then(Value::as_str) { - text.push_str(value); - } - } - if let Some(Value::Array(content)) = result.get("content") { - for block in content { - if let Some(value) = block.get("text").and_then(Value::as_str) { - text.push_str(value); - } - } - } - if let Some(citations) = result.get("citations") { - text.push_str(&citations.to_string()); - } - } - text -} - -fn collect_input_text(value: &Value, parts: &mut Vec) { - match value { - Value::String(text) => { - let trimmed = text.trim(); - if !trimmed.is_empty() { - parts.push(trimmed.to_string()); - } - } - Value::Array(items) => { - for item in items { - collect_input_text(item, parts); - } - } - Value::Object(map) => { - if let Some(content) = map.get("content").filter(|content| !content.is_null()) { - collect_input_text(content, parts); - return; - } - for key in ["text", "output", "input_text", "output_text"] { - if let Some(Value::String(text)) = map.get(key) { - let trimmed = text.trim(); - if !trimmed.is_empty() { - parts.push(trimmed.to_string()); - return; - } - } - } - } - _ => {} - } -} diff --git a/litellm-rust/crates/cache-redis-semantic/src/reply.rs b/litellm-rust/crates/cache-redis-semantic/src/reply.rs new file mode 100644 index 00000000000..24cbd573cde --- /dev/null +++ b/litellm-rust/crates/cache-redis-semantic/src/reply.rs @@ -0,0 +1,57 @@ +pub(crate) fn string_value(value: &redis::Value) -> Option { + match value { + redis::Value::BulkString(bytes) => String::from_utf8(bytes.clone()).ok(), + redis::Value::SimpleString(text) => Some(text.clone()), + redis::Value::VerbatimString { text, .. } => Some(text.clone()), + _ => None, + } +} + +pub(crate) fn number_value(value: &redis::Value) -> Option { + match value { + redis::Value::Int(number) => Some(*number as f64), + redis::Value::Double(number) => Some(*number), + _ => string_value(value).and_then(|text| text.parse().ok()), + } +} + +pub(crate) fn first_document(result: &redis::Value) -> Option<&[redis::Value]> { + let redis::Value::Array(items) = result else { + return None; + }; + let [count, _document_id, fields, ..] = items.as_slice() else { + return None; + }; + if !matches!(count, redis::Value::Int(count) if *count > 0) { + return None; + } + match fields { + redis::Value::Array(fields) => Some(fields.as_slice()), + _ => None, + } +} + +fn field_value<'a>(fields: &'a [redis::Value], name: &str) -> Option<&'a redis::Value> { + fields + .as_chunks::<2>() + .0 + .iter() + .find(|pair| string_value(&pair[0]).as_deref() == Some(name)) + .map(|pair| &pair[1]) +} + +pub(crate) fn string_field(fields: &[redis::Value], name: &str) -> Option { + field_value(fields, name).and_then(string_value) +} + +pub(crate) fn number_field(fields: &[redis::Value], name: &str) -> Option { + field_value(fields, name).and_then(number_value) +} + +pub(crate) fn bytes_field(fields: &[redis::Value], name: &str) -> Option> { + match field_value(fields, name)? { + redis::Value::BulkString(bytes) => Some(bytes.clone()), + redis::Value::SimpleString(text) => Some(text.clone().into_bytes()), + _ => None, + } +} diff --git a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs index 233b87ec52f..fa4ac00e767 100644 --- a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs +++ b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs @@ -1,55 +1,26 @@ -use std::{ - collections::HashMap, - sync::{Arc, Mutex}, - time::Duration, -}; +mod support; -use litellm_cache::{BaseCache, CacheCodec, Error, SemanticCacheContext}; -use litellm_cache_redis_semantic::{Embedder, RedisSemanticCache, RedisSemanticConfig}; -use litellm_cache_response::{CacheEntry, ResponseCacheCodec}; +use std::time::Duration; + +use litellm_cache::{ + BaseCache, Error, JsonCodec, SemanticCacheContext, + semantic::{SemanticCache, SemanticLookup}, +}; +use litellm_cache_redis_semantic::{DEFAULT_INDEX_NAME, RedisSemanticCache, RedisSemanticConfig}; use redis_test::{MockCmd, MockRedisConnection}; +use rstest::{fixture, rstest}; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; +use support::FakeEmbedder; -const INDEX: &str = "litellm_semantic_cache_index"; +const INDEX: &str = DEFAULT_INDEX_NAME; +const PROMPT: &str = "hello prompt"; +const CLOCK: fn() -> f64 = || 1700000000.5; +const VECTOR: [f32; 3] = [0.1, 0.2, 0.3]; -struct FakeEmbedder { - vectors: HashMap>, - calls: Arc>>, -} - -impl FakeEmbedder { - fn new(vectors: &[(&str, &[f32])]) -> (Self, Arc>>) { - let calls = Arc::new(Mutex::new(Vec::new())); - ( - Self { - vectors: vectors - .iter() - .map(|(prompt, vector)| (prompt.to_string(), vector.to_vec())) - .collect(), - calls: Arc::clone(&calls), - }, - calls, - ) - } -} - -impl Embedder for FakeEmbedder { - fn embed(&self, prompt: &str, _: Option<&Value>) -> Result, Error> { - self.calls.lock().unwrap().push(prompt.to_string()); - - Ok(self - .vectors - .get(prompt) - .cloned() - .unwrap_or_else(|| vec![0.1, 0.2, 0.3])) - } - - async fn async_embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { - self.embed(prompt, metadata) - } -} +type MockCache = RedisSemanticCache, MockRedisConnection>; +#[fixture] fn config() -> RedisSemanticConfig { RedisSemanticConfig { index_name: INDEX.into(), @@ -57,6 +28,16 @@ fn config() -> RedisSemanticConfig { } } +#[fixture] +fn entry() -> Value { + json!({"timestamp": 1.0, "response": {"answer": "yes"}}) +} + +#[fixture] +fn context() -> SemanticCacheContext { + messages_context(vec![json!({"role": "user", "content": PROMPT})]) +} + fn messages_context(messages: Vec) -> SemanticCacheContext { SemanticCacheContext { messages: Some(Value::Array(messages)), @@ -64,15 +45,18 @@ fn messages_context(messages: Vec) -> SemanticCacheContext { } } -fn entry() -> CacheEntry { - CacheEntry { - timestamp: Some(1.0), - response: json!({"answer": "yes"}), - } +fn cache(commands: Vec, embedder: FakeEmbedder) -> MockCache { + RedisSemanticCache::with_connection( + MockRedisConnection::new(commands).assert_all_commands_consumed(), + embedder, + JsonCodec::new(), + config(), + ) + .with_clock(CLOCK) } -fn encoded(entry: &CacheEntry) -> Vec { - ResponseCacheCodec.encode(entry).unwrap() +fn encoded(value: &Value) -> Vec { + serde_json::to_vec(value).unwrap() } fn vector_bytes(vector: &[f32]) -> Vec { @@ -98,6 +82,17 @@ fn unknown_index_error() -> redis::RedisError { redis::RedisError::from((redis::ErrorKind::Extension, "Unknown index name")) } +fn info_missing(index: &str) -> MockCmd { + MockCmd::new( + redis::cmd("FT.INFO").arg(index), + Err::(unknown_index_error()), + ) +} + +fn info(index: &str, value: redis::Value) -> MockCmd { + MockCmd::new(redis::cmd("FT.INFO").arg(index), Ok(value)) +} + fn attribute(name: &str, field_type: &str, extra: Vec) -> redis::Value { let mut parts = vec![ s("identifier"), @@ -137,10 +132,6 @@ fn vector_attribute_with(dims: i64, data_type: &str, distance_metric: &str) -> r ) } -fn vector_attribute(dims: i64) -> redis::Value { - vector_attribute_with(dims, "FLOAT32", "COSINE") -} - fn info_with_vector(vector: redis::Value) -> redis::Value { index_info(vec![ attribute("prompt", "TEXT", vec![]), @@ -153,7 +144,7 @@ fn info_with_vector(vector: redis::Value) -> redis::Value { } fn compatible_info(dims: i64) -> redis::Value { - info_with_vector(vector_attribute(dims)) + info_with_vector(vector_attribute_with(dims, "FLOAT32", "COSINE")) } fn unscoped_info(dims: i64) -> redis::Value { @@ -162,10 +153,14 @@ fn unscoped_info(dims: i64) -> redis::Value { attribute("response", "TEXT", vec![]), attribute("inserted_at", "NUMERIC", vec![]), attribute("updated_at", "NUMERIC", vec![]), - vector_attribute(dims), + vector_attribute_with(dims, "FLOAT32", "COSINE"), ]) } +fn create_index(name: &str, dims: usize) -> MockCmd { + MockCmd::new(create_index_command(name, dims), Ok("OK")) +} + fn create_index_command(name: &str, dims: usize) -> redis::Cmd { let mut command = redis::cmd("FT.CREATE"); command @@ -207,7 +202,34 @@ fn create_index_command(name: &str, dims: usize) -> redis::Cmd { command } -fn search_command(index: &str, tag: &str, vector: &[f32]) -> redis::Cmd { +fn hset(index: &str, prompt: &str, tag: &str, vector: &[f32], value: &Value) -> MockCmd { + MockCmd::new( + redis::cmd("HSET") + .arg(format!("{index}:{}", entry_id(prompt, tag))) + .arg("entry_id") + .arg(entry_id(prompt, tag)) + .arg("prompt") + .arg(prompt) + .arg("response") + .arg(encoded(value)) + .arg("prompt_vector") + .arg(vector_bytes(vector)) + .arg("inserted_at") + .arg("1700000000.5") + .arg("updated_at") + .arg("1700000000.5") + .arg("litellm_cache_key") + .arg(tag), + Ok(7), + ) +} + +fn search( + index: &str, + tag: &str, + vector: &[f32], + reply: redis::RedisResult, +) -> MockCmd { let mut command = redis::cmd("FT.SEARCH"); command .arg(index) @@ -236,33 +258,29 @@ fn search_command(index: &str, tag: &str, vector: &[f32]) -> redis::Cmd { .arg(2) .arg("vector") .arg(vector_bytes(vector)); - command + MockCmd::new(command, reply) } -fn hit_fields(tag: &str, distance: &str, response: Vec) -> redis::Value { - redis::Value::Array(vec![ - s("entry_id"), - s("stored-id"), - s("prompt"), - s("hello prompt"), - s("response"), - redis::Value::BulkString(response), - s("inserted_at"), - s("1700000000.5"), - s("updated_at"), - s("1700000000.5"), - s("litellm_cache_key"), - s(tag), - s("vector_distance"), - s(distance), - ]) -} - -fn search_result(fields: redis::Value) -> redis::Value { +fn hit(tag: &str, distance: &str, response: Vec) -> redis::Value { redis::Value::Array(vec![ redis::Value::Int(1), s("litellm_semantic_cache_index:stored-id"), - fields, + redis::Value::Array(vec![ + s("entry_id"), + s("stored-id"), + s("prompt"), + s(PROMPT), + s("response"), + redis::Value::BulkString(response), + s("inserted_at"), + s("1700000000.5"), + s("updated_at"), + s("1700000000.5"), + s("litellm_cache_key"), + s(tag), + s("vector_distance"), + s(distance), + ]), ]) } @@ -270,685 +288,452 @@ fn empty_result() -> redis::Value { redis::Value::Array(vec![redis::Value::Int(0)]) } -#[test] -fn store_creates_index_and_writes_hash_with_expire() { - let vector = vec![0.1f32, 0.2, 0.3]; - let prompt = "hello prompt"; - let tag = "key1"; - let hash_key = format!("{INDEX}:{}", entry_id(prompt, tag)); - let value = entry(); - let connection = MockRedisConnection::new([ - MockCmd::new( - redis::cmd("FT.INFO").arg(INDEX), - Err::(unknown_index_error()), - ), - MockCmd::new(create_index_command(INDEX, 3), Ok("OK")), - MockCmd::new( - redis::cmd("HSET") - .arg(&hash_key) - .arg("entry_id") - .arg(entry_id(prompt, tag)) - .arg("prompt") - .arg(prompt) - .arg("response") - .arg(encoded(&value)) - .arg("prompt_vector") - .arg(vector_bytes(&vector)) - .arg("inserted_at") - .arg("1700000000.5") - .arg("updated_at") - .arg("1700000000.5") - .arg("litellm_cache_key") - .arg(tag), - Ok(7), - ), - MockCmd::new(redis::cmd("EXPIRE").arg(&hash_key).arg(5), Ok(1)), - ]) - .assert_all_commands_consumed(); - let (embedder, _) = FakeEmbedder::new(&[(prompt, &vector)]); - let cache = RedisSemanticCache::with_connection(connection, embedder, config()) - .with_clock(|| 1700000000.5); - - let context = SemanticCacheContext { - ttl: Some(Duration::from_secs(5)), - ..messages_context(vec![json!({"role": "user", "content": prompt})]) +#[rstest] +#[case::creates_index_and_expires(false, Some(Duration::from_secs(5)), Some(5))] +#[case::existing_index_without_ttl(true, None, None)] +#[case::fractional_ttl_rounds_up(true, Some(Duration::from_millis(1500)), Some(2))] +fn store_writes_the_redisvl_hash( + #[case] index_exists: bool, + #[case] ttl: Option, + #[case] expire: Option, + entry: Value, + context: SemanticCacheContext, +) { + let hash_key = format!("{INDEX}:{}", entry_id(PROMPT, "key1")); + let mut commands = if index_exists { + vec![info(INDEX, compatible_info(3))] + } else { + vec![info_missing(INDEX), create_index(INDEX, 3)] }; - cache.set_cache(tag, value, &context).unwrap(); -} - -#[test] -fn store_without_ttl_skips_expire() { - let prompt = "hello prompt"; - let connection = MockRedisConnection::new([ - MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))), - MockCmd::new( - redis::cmd("HSET") - .arg(format!("{INDEX}:{}", entry_id(prompt, "key1"))) - .arg("entry_id") - .arg(entry_id(prompt, "key1")) - .arg("prompt") - .arg(prompt) - .arg("response") - .arg(encoded(&entry())) - .arg("prompt_vector") - .arg(vector_bytes(&[0.1f32, 0.2, 0.3])) - .arg("inserted_at") - .arg("1700000000.5") - .arg("updated_at") - .arg("1700000000.5") - .arg("litellm_cache_key") - .arg("key1"), - Ok(7), - ), - ]) - .assert_all_commands_consumed(); - let (embedder, _) = FakeEmbedder::new(&[]); - let cache = RedisSemanticCache::with_connection(connection, embedder, config()) - .with_clock(|| 1700000000.5); + commands.push(hset(INDEX, PROMPT, "key1", &VECTOR, &entry)); + commands.extend( + expire.map(|seconds| MockCmd::new(redis::cmd("EXPIRE").arg(&hash_key).arg(seconds), Ok(1))), + ); + let cache = cache(commands, FakeEmbedder::new(&[])); cache - .set_cache( - "key1", - entry(), - &messages_context(vec![json!({"role": "user", "content": prompt})]), - ) + .set_cache("key1", entry, &SemanticCacheContext { ttl, ..context }) .unwrap(); } -#[test] -fn lookup_returns_hit_below_distance_threshold() { - let vector = vec![0.1f32, 0.2, 0.3]; - let value = entry(); - let connection = MockRedisConnection::new([ - MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))), - MockCmd::new( - search_command(INDEX, "key1", &vector), - Ok(search_result(hit_fields("key1", "0.05", encoded(&value)))), - ), - ]) - .assert_all_commands_consumed(); - let (embedder, _) = FakeEmbedder::new(&[]); - let cache = RedisSemanticCache::with_connection(connection, embedder, config()); - - let hit = cache - .get_cache( - "key1", - &messages_context(vec![json!({"role": "user", "content": "hello prompt"})]), - ) - .unwrap(); - assert_eq!(hit, Some(value)); -} - -#[test] -fn lookup_misses_above_distance_threshold_and_on_tag_mismatch() { - let vector = vec![0.1f32, 0.2, 0.3]; - let connection = MockRedisConnection::new([ - MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))), - MockCmd::new( - search_command(INDEX, "key1", &vector), - Ok(search_result(hit_fields("key1", "0.5", encoded(&entry())))), - ), - MockCmd::new( - search_command(INDEX, "key1", &vector), - Ok(search_result(hit_fields( - "other", - "0.05", - encoded(&entry()), - ))), - ), - ]) - .assert_all_commands_consumed(); - let (embedder, _) = FakeEmbedder::new(&[]); - let cache = RedisSemanticCache::with_connection(connection, embedder, config()); - let context = messages_context(vec![json!({"role": "user", "content": "hello prompt"})]); - - assert_eq!(cache.get_cache("key1", &context).unwrap(), None); - assert_eq!(cache.get_cache("key1", &context).unwrap(), None); -} - -#[test] -fn lookup_returns_invalid_entry_on_malformed_response() { - let vector = vec![0.1f32, 0.2, 0.3]; - let connection = MockRedisConnection::new([ - MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))), - MockCmd::new( - search_command(INDEX, "key1", &vector), - Ok(search_result(hit_fields( +#[rstest] +#[case::below_distance_threshold("key1", "0.05", None, Ok(Some(entry())))] +#[case::above_distance_threshold("key1", "0.5", None, Ok(None))] +#[case::other_cache_key("other", "0.05", None, Ok(None))] +#[case::malformed_response("key1", "0.05", Some(b"not json!".as_slice()), Err(Error::InvalidEntry))] +fn lookup_applies_threshold_scope_and_codec( + #[case] stored_tag: &str, + #[case] distance: &str, + #[case] response: Option<&[u8]>, + #[case] expected: Result, Error>, + entry: Value, + context: SemanticCacheContext, +) { + let response = response.map_or_else(|| encoded(&entry), <[u8]>::to_vec); + let cache = cache( + vec![ + info(INDEX, compatible_info(3)), + search( + INDEX, "key1", - "0.05", - b"not json!".to_vec(), - ))), - ), - ]) - .assert_all_commands_consumed(); - let (embedder, _) = FakeEmbedder::new(&[]); - let cache = RedisSemanticCache::with_connection(connection, embedder, config()); - - assert_eq!( - cache - .get_cache( - "key1", - &messages_context(vec![json!({"role": "user", "content": "hello prompt"})]) - ) - .unwrap_err(), - Error::InvalidEntry + &VECTOR, + Ok(hit(stored_tag, distance, response)), + ), + ], + FakeEmbedder::new(&[]), ); + + assert_eq!(cache.get_cache("key1", &context), expected); } -#[test] -fn missing_prompt_is_noop_and_never_embeds() { - let connection = MockRedisConnection::new(Vec::::new()).assert_all_commands_consumed(); - let (embedder, calls) = FakeEmbedder::new(&[]); - let cache = RedisSemanticCache::with_connection(connection, embedder, config()); +#[rstest] +#[case::hit(context(), Some(hit("key1", "0.05", encoded(&entry()))), Some(entry()), Some(1.0 - 0.05))] +#[case::beyond_distance_threshold(context(), Some(hit("key1", "0.5", encoded(&entry()))), None, Some(0.0))] +#[case::no_results(context(), Some(empty_result()), None, Some(0.0))] +#[case::other_cache_key(context(), Some(hit("other", "0.05", encoded(&entry()))), None, Some(0.0))] +#[case::no_prompt(SemanticCacheContext::default(), None, None, Some(0.0))] +#[tokio::test] +async fn lookup_reports_python_semantic_similarity( + #[case] context: SemanticCacheContext, + #[case] reply: Option, + #[case] value: Option, + #[case] similarity: Option, + #[values(false, true)] use_async: bool, +) { + let commands = reply.map_or_else(Vec::new, |reply| { + vec![ + info(INDEX, compatible_info(3)), + search(INDEX, "key1", &VECTOR, Ok(reply)), + ] + }); + let cache = cache(commands, FakeEmbedder::new(&[])); + let lookup = if use_async { + cache + .async_get_cache_with_similarity("key1", &context) + .await + } else { + cache.get_cache_with_similarity("key1", &context) + }; + + assert_eq!(lookup, Ok(SemanticLookup { value, similarity })); +} + +#[rstest] +#[tokio::test] +async fn missing_prompt_is_a_noop_that_never_embeds(entry: Value) { + let embedder = FakeEmbedder::new(&[]); + let calls = embedder.calls.clone(); + let cache = cache(Vec::new(), embedder); let context = SemanticCacheContext::default(); - cache.set_cache("key1", entry(), &context).unwrap(); + + cache.set_cache("key1", entry.clone(), &context).unwrap(); assert_eq!(cache.get_cache("key1", &context).unwrap(), None); + cache + .async_set_cache("key1", entry, context.clone()) + .await + .unwrap(); + assert_eq!(cache.async_get_cache("key1", &context).await.unwrap(), None); assert!(calls.lock().unwrap().is_empty()); } -#[test] -fn scope_overrides_key_as_filter_tag() { - let vector = vec![0.1f32, 0.2, 0.3]; - let prompt = "hello prompt"; - let value = entry(); - let connection = MockRedisConnection::new([ - MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))), - MockCmd::new( - redis::cmd("HSET") - .arg(format!("{INDEX}:{}", entry_id(prompt, "scope-a"))) - .arg("entry_id") - .arg(entry_id(prompt, "scope-a")) - .arg("prompt") - .arg(prompt) - .arg("response") - .arg(encoded(&value)) - .arg("prompt_vector") - .arg(vector_bytes(&vector)) - .arg("inserted_at") - .arg("1700000000.5") - .arg("updated_at") - .arg("1700000000.5") - .arg("litellm_cache_key") - .arg("scope-a"), - Ok(7), - ), - MockCmd::new( - search_command(INDEX, "scope\\-a", &vector), - Ok(search_result(hit_fields( - "scope-a", - "0.05", - encoded(&value), - ))), - ), - ]) - .assert_all_commands_consumed(); - let (embedder, _) = FakeEmbedder::new(&[]); - let cache = RedisSemanticCache::with_connection(connection, embedder, config()) - .with_clock(|| 1700000000.5); +#[rstest] +fn scope_overrides_key_as_filter_tag(entry: Value, context: SemanticCacheContext) { + let cache = cache( + vec![ + info(INDEX, compatible_info(3)), + hset(INDEX, PROMPT, "scope-a", &VECTOR, &entry), + search( + INDEX, + "scope\\-a", + &VECTOR, + Ok(hit("scope-a", "0.05", encoded(&entry))), + ), + ], + FakeEmbedder::new(&[]), + ); let context = SemanticCacheContext { scope: Some("scope-a".into()), - ..messages_context(vec![json!({"role": "user", "content": prompt})]) + ..context }; - cache.set_cache("key1", value.clone(), &context).unwrap(); - assert_eq!(cache.get_cache("key1", &context).unwrap(), Some(value)); + cache.set_cache("key1", entry.clone(), &context).unwrap(); + assert_eq!(cache.get_cache("key1", &context).unwrap(), Some(entry)); } -#[test] -fn incompatible_schema_falls_back_to_isolated_index() { - let prompt = "hello prompt"; - let tag = "key1"; +#[rstest] +#[case::unscoped_schema(unscoped_info(3))] +#[case::wrong_distance_metric(info_with_vector(vector_attribute_with(3, "FLOAT32", "L2")))] +#[case::wrong_data_type(info_with_vector(vector_attribute_with(3, "FLOAT64", "COSINE")))] +fn incompatible_schema_falls_back_to_isolated_index( + #[case] base_info: redis::Value, + entry: Value, + context: SemanticCacheContext, +) { let isolated = format!("{INDEX}_isolated"); - let connection = MockRedisConnection::new([ - MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(unscoped_info(3))), - MockCmd::new( - redis::cmd("FT.INFO").arg(&isolated), - Err::(unknown_index_error()), - ), - MockCmd::new(create_index_command(&isolated, 3), Ok("OK")), - MockCmd::new( - redis::cmd("HSET") - .arg(format!("{isolated}:{}", entry_id(prompt, tag))) - .arg("entry_id") - .arg(entry_id(prompt, tag)) - .arg("prompt") - .arg(prompt) - .arg("response") - .arg(encoded(&entry())) - .arg("prompt_vector") - .arg(vector_bytes(&[0.1f32, 0.2, 0.3])) - .arg("inserted_at") - .arg("1700000000.5") - .arg("updated_at") - .arg("1700000000.5") - .arg("litellm_cache_key") - .arg(tag), - Ok(7), - ), - ]) - .assert_all_commands_consumed(); - let (embedder, _) = FakeEmbedder::new(&[]); - let cache = RedisSemanticCache::with_connection(connection, embedder, config()) - .with_clock(|| 1700000000.5); - - cache - .set_cache( - tag, - entry(), - &messages_context(vec![json!({"role": "user", "content": prompt})]), - ) - .unwrap(); -} - -#[test] -fn create_index_race_rechecks_schema_and_stores() { - let prompt = "hello prompt"; - let tag = "key1"; - let connection = MockRedisConnection::new([ - MockCmd::new( - redis::cmd("FT.INFO").arg(INDEX), - Err::(unknown_index_error()), - ), - MockCmd::new( - create_index_command(INDEX, 3), - Err::<&str, _>(redis::RedisError::from(( - redis::ErrorKind::Extension, - "Index already exists", - ))), - ), - MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))), - MockCmd::new( - redis::cmd("HSET") - .arg(format!("{INDEX}:{}", entry_id(prompt, tag))) - .arg("entry_id") - .arg(entry_id(prompt, tag)) - .arg("prompt") - .arg(prompt) - .arg("response") - .arg(encoded(&entry())) - .arg("prompt_vector") - .arg(vector_bytes(&[0.1f32, 0.2, 0.3])) - .arg("inserted_at") - .arg("1700000000.5") - .arg("updated_at") - .arg("1700000000.5") - .arg("litellm_cache_key") - .arg(tag), - Ok(7), - ), - ]) - .assert_all_commands_consumed(); - let (embedder, _) = FakeEmbedder::new(&[]); - let cache = RedisSemanticCache::with_connection(connection, embedder, config()) - .with_clock(|| 1700000000.5); - - cache - .set_cache( - tag, - entry(), - &messages_context(vec![json!({"role": "user", "content": prompt})]), - ) - .unwrap(); -} - -#[test] -fn wrong_distance_metric_falls_back_to_isolated_index() { - let prompt = "hello prompt"; - let tag = "key1"; - let isolated = format!("{INDEX}_isolated"); - let connection = MockRedisConnection::new([ - MockCmd::new( - redis::cmd("FT.INFO").arg(INDEX), - Ok(info_with_vector(vector_attribute_with(3, "FLOAT32", "L2"))), - ), - MockCmd::new( - redis::cmd("FT.INFO").arg(&isolated), - Err::(unknown_index_error()), - ), - MockCmd::new(create_index_command(&isolated, 3), Ok("OK")), - MockCmd::new( - redis::cmd("HSET") - .arg(format!("{isolated}:{}", entry_id(prompt, tag))) - .arg("entry_id") - .arg(entry_id(prompt, tag)) - .arg("prompt") - .arg(prompt) - .arg("response") - .arg(encoded(&entry())) - .arg("prompt_vector") - .arg(vector_bytes(&[0.1f32, 0.2, 0.3])) - .arg("inserted_at") - .arg("1700000000.5") - .arg("updated_at") - .arg("1700000000.5") - .arg("litellm_cache_key") - .arg(tag), - Ok(7), - ), - ]) - .assert_all_commands_consumed(); - let (embedder, _) = FakeEmbedder::new(&[]); - let cache = RedisSemanticCache::with_connection(connection, embedder, config()) - .with_clock(|| 1700000000.5); - - cache - .set_cache( - tag, - entry(), - &messages_context(vec![json!({"role": "user", "content": prompt})]), - ) - .unwrap(); -} - -#[test] -fn tag_special_characters_are_escaped_in_search_filter() { - let vector = vec![0.1f32, 0.2, 0.3]; - let tag = "a:b, c|d"; - let connection = MockRedisConnection::new([ - MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))), - MockCmd::new( - search_command(INDEX, "a\\:b\\,\\ c\\|d", &vector), - Ok(empty_result()), - ), - ]) - .assert_all_commands_consumed(); - let (embedder, _) = FakeEmbedder::new(&[]); - let cache = RedisSemanticCache::with_connection(connection, embedder, config()); - - assert_eq!( - cache - .get_cache( - tag, - &messages_context(vec![json!({"role": "user", "content": "hello prompt"})]) - ) - .unwrap(), - None - ); -} - -#[test] -fn prompt_extraction_matches_python_message_and_input_shapes() { - let vector = vec![0.1f32, 0.2, 0.3]; - let lookups = 5; - let mut commands = vec![MockCmd::new( - redis::cmd("FT.INFO").arg(INDEX), - Ok(compatible_info(3)), - )]; - for _ in 0..lookups { - commands.push(MockCmd::new( - search_command(INDEX, "key1", &vector), - Ok(empty_result()), - )); - } - let connection = MockRedisConnection::new(commands).assert_all_commands_consumed(); - let (embedder, calls) = FakeEmbedder::new(&[]); - let cache = RedisSemanticCache::with_connection(connection, embedder, config()); - - cache - .get_cache( - "key1", - &messages_context(vec![ - json!({"role": "user", "content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}]}), - json!({"role": "assistant", "content": "reply"}), - ]), - ) - .unwrap(); - cache - .get_cache( - "key1", - &SemanticCacheContext { - input: Some(json!(" plain input ")), - ..Default::default() - }, - ) - .unwrap(); - cache - .get_cache( - "key1", - &SemanticCacheContext { - input: Some( - json!([{"content": [{"type": "input_text", "text": "nested"}]}, "tail"]), - ), - ..Default::default() - }, - ) - .unwrap(); - cache - .get_cache( - "key1", - &SemanticCacheContext { - input: Some(json!({"output_text": " result text "})), - ..Default::default() - }, - ) - .unwrap(); - cache - .get_cache( - "key1", - &messages_context(vec![json!({ - "role": "user", - "content": "question", - "search_results": [{"source": "src", "title": "t", "content": [{"text": "found"}], "citations": {"a": 1}}], - })]), - ) - .unwrap(); - - assert_eq!( - *calls.lock().unwrap(), + let cache = cache( vec![ - "firstsecondreply", - "plain input", - "nested\ntail", - "result text", - "questionsrctfound{\"a\":1}", - ] + info(INDEX, base_info), + info_missing(&isolated), + create_index(&isolated, 3), + hset(&isolated, PROMPT, "key1", &VECTOR, &entry), + ], + FakeEmbedder::new(&[]), ); + + cache.set_cache("key1", entry, &context).unwrap(); } -#[test] -fn ttl_passes_through_context_only() { - let (embedder, _) = FakeEmbedder::new(&[]); - let cache = RedisSemanticCache::with_connection( - MockRedisConnection::new(Vec::::new()), - embedder, - config(), +#[rstest] +fn create_index_race_rechecks_schema_and_stores(entry: Value, context: SemanticCacheContext) { + let cache = cache( + vec![ + info_missing(INDEX), + MockCmd::new( + create_index_command(INDEX, 3), + Err::<&str, _>(redis::RedisError::from(( + redis::ErrorKind::Extension, + "Index already exists", + ))), + ), + info(INDEX, compatible_info(3)), + hset(INDEX, PROMPT, "key1", &VECTOR, &entry), + ], + FakeEmbedder::new(&[]), ); - assert_eq!(cache.get_ttl(&SemanticCacheContext::default()), None); + + cache.set_cache("key1", entry, &context).unwrap(); +} + +#[rstest] +#[case::punctuation_and_spaces("a:b, c|d", "a\\:b\\,\\ c\\|d")] +#[case::braces_and_dots("{x}.y", "\\{x\\}\\.y")] +#[case::plain("key1", "key1")] +fn tag_special_characters_are_escaped_in_search_filter( + #[case] tag: &str, + #[case] escaped: &str, + context: SemanticCacheContext, +) { + let cache = cache( + vec![ + info(INDEX, compatible_info(3)), + search(INDEX, escaped, &VECTOR, Ok(empty_result())), + ], + FakeEmbedder::new(&[]), + ); + + assert_eq!(cache.get_cache(tag, &context).unwrap(), None); +} + +#[rstest] +#[case::content_parts( + SemanticCacheContext { + messages: Some(json!([ + {"role": "user", "content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}]}, + {"role": "assistant", "content": "reply"}, + ])), + ..Default::default() + }, + "firstsecondreply" +)] +#[case::responses_string_input( + SemanticCacheContext { input: Some(json!(" plain input ")), ..Default::default() }, + "plain input" +)] +#[case::responses_nested_input( + SemanticCacheContext { + input: Some(json!([{"content": [{"type": "input_text", "text": "nested"}]}, "tail"])), + ..Default::default() + }, + "nested\ntail" +)] +#[case::responses_output_text( + SemanticCacheContext { input: Some(json!({"output_text": " result text "})), ..Default::default() }, + "result text" +)] +#[case::search_results( + messages_context(vec![json!({ + "role": "user", + "content": "question", + "search_results": [{"source": "src", "title": "t", "content": [{"text": "found"}], "citations": {"a": 1}}], + })]), + "questionsrctfound{\"a\":1}" +)] +#[case::empty_messages_fall_back_to_input( + SemanticCacheContext { messages: Some(json!([])), input: Some(json!("fallback")), ..Default::default() }, + "fallback" +)] +fn prompt_extraction_matches_python_message_and_input_shapes( + #[case] context: SemanticCacheContext, + #[case] prompt: &str, +) { + let embedder = FakeEmbedder::new(&[]); + let calls = embedder.calls.clone(); + let cache = cache( + vec![ + info(INDEX, compatible_info(3)), + search(INDEX, "key1", &VECTOR, Ok(empty_result())), + ], + embedder, + ); + + cache.get_cache("key1", &context).unwrap(); + + assert_eq!(*calls.lock().unwrap(), vec![(prompt.to_owned(), None)]); +} + +#[rstest] +#[case(None)] +#[case(Some(Duration::from_secs(9)))] +fn ttl_passes_through_context_only(#[case] ttl: Option) { + let cache = cache(Vec::new(), FakeEmbedder::new(&[])); + assert_eq!( cache.get_ttl(&SemanticCacheContext { - ttl: Some(Duration::from_secs(9)), + ttl, ..Default::default() }), - Some(Duration::from_secs(9)) + ttl ); } +#[rstest] #[tokio::test] -async fn async_paths_embed_then_run_blocking_redis_work() { - let vector = vec![0.1f32, 0.2, 0.3]; - let prompt = "hello prompt"; - let tag = "key1"; - let hash_key = format!("{INDEX}:{}", entry_id(prompt, tag)); - let value = entry(); - let connection = MockRedisConnection::new([ - MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(3))), - MockCmd::new( - redis::cmd("HSET") - .arg(&hash_key) - .arg("entry_id") - .arg(entry_id(prompt, tag)) - .arg("prompt") - .arg(prompt) - .arg("response") - .arg(encoded(&value)) - .arg("prompt_vector") - .arg(vector_bytes(&vector)) - .arg("inserted_at") - .arg("1700000000.5") - .arg("updated_at") - .arg("1700000000.5") - .arg("litellm_cache_key") - .arg(tag), - Ok(7), - ), - MockCmd::new( - search_command(INDEX, tag, &vector), - Ok(search_result(hit_fields(tag, "0.05", encoded(&value)))), - ), - ]) - .assert_all_commands_consumed(); - let (embedder, _) = FakeEmbedder::new(&[]); - let cache = RedisSemanticCache::with_connection(connection, embedder, config()) - .with_clock(|| 1700000000.5); - let context = messages_context(vec![json!({"role": "user", "content": prompt})]); +async fn async_paths_embed_with_metadata_then_run_blocking_redis_work( + entry: Value, + context: SemanticCacheContext, +) { + let embedder = FakeEmbedder::new(&[]); + let calls = embedder.calls.clone(); + let cache = cache( + vec![ + info(INDEX, compatible_info(3)), + hset(INDEX, PROMPT, "key1", &VECTOR, &entry), + search( + INDEX, + "key1", + &VECTOR, + Ok(hit("key1", "0.05", encoded(&entry))), + ), + ], + embedder, + ); + let context = SemanticCacheContext { + metadata: Some(json!({"tenant": "team"})), + ..context + }; cache - .async_set_cache(tag, value.clone(), context.clone()) + .async_set_cache("key1", entry.clone(), context.clone()) .await .unwrap(); assert_eq!( - cache.async_get_cache(tag, &context).await.unwrap(), - Some(value) + cache.async_get_cache("key1", &context).await.unwrap(), + Some(entry) + ); + assert_eq!( + *calls.lock().unwrap(), + vec![(PROMPT.to_owned(), context.metadata.clone()); 2] ); } -#[test] -fn shared_base_index_across_dimensions_replaces_the_isolated_index() { +#[rstest] +fn accessors_report_the_config(config: RedisSemanticConfig) { + let cache = cache(Vec::new(), FakeEmbedder::new(&[])); + + assert_eq!(cache.index_name(), config.index_name); + assert!((cache.similarity_threshold() - config.similarity_threshold).abs() < 1e-6); +} + +#[rstest] +fn shared_base_index_across_dimensions_replaces_the_isolated_index(entry: Value) { // Pins parity with Python's `_isolated` + overwrite=True flow. let prompt = "shared prompt"; - let tag = "key1"; let isolated = format!("{INDEX}_isolated"); - let value = entry(); let context = || messages_context(vec![json!({"role": "user", "content": prompt})]); - let store_hash = |index: &str, vector: &[f32]| { - MockCmd::new( - redis::cmd("HSET") - .arg(format!("{index}:{}", entry_id(prompt, tag))) - .arg("entry_id") - .arg(entry_id(prompt, tag)) - .arg("prompt") - .arg(prompt) - .arg("response") - .arg(encoded(&value)) - .arg("prompt_vector") - .arg(vector_bytes(vector)) - .arg("inserted_at") - .arg("1700000000.5") - .arg("updated_at") - .arg("1700000000.5") - .arg("litellm_cache_key") - .arg(tag), - Ok(7), - ) - }; let vector_a = vec![0.1f32; 8]; - let connection_a = MockRedisConnection::new([ - MockCmd::new( - redis::cmd("FT.INFO").arg(INDEX), - Err::(unknown_index_error()), - ), - MockCmd::new(create_index_command(INDEX, 8), Ok("OK")), - store_hash(INDEX, &vector_a), - ]) - .assert_all_commands_consumed(); - let (embedder_a, _) = FakeEmbedder::new(&[(prompt, &vector_a)]); - let worker_a = RedisSemanticCache::with_connection(connection_a, embedder_a, config()) - .with_clock(|| 1700000000.5); - worker_a.set_cache(tag, value.clone(), &context()).unwrap(); + let worker_a = cache( + vec![ + info_missing(INDEX), + create_index(INDEX, 8), + hset(INDEX, prompt, "key1", &vector_a, &entry), + ], + FakeEmbedder::new(&[(prompt, &vector_a)]), + ); + worker_a + .set_cache("key1", entry.clone(), &context()) + .unwrap(); let vector_b = vec![0.2f32; 4]; - let connection_b = MockRedisConnection::new([ - MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(8))), - MockCmd::new( - redis::cmd("FT.INFO").arg(&isolated), - Err::(unknown_index_error()), - ), - MockCmd::new(create_index_command(&isolated, 4), Ok("OK")), - store_hash(&isolated, &vector_b), - MockCmd::new( - search_command(&isolated, tag, &vector_b), - Ok(search_result(hit_fields(tag, "0.0", encoded(&value)))), - ), - MockCmd::new( - search_command(&isolated, tag, &vector_b), - Err::(redis::RedisError::from(( - redis::ErrorKind::Extension, - "Vector dimension mismatch", - ))), - ), - ]) - .assert_all_commands_consumed(); - let (embedder_b, _) = FakeEmbedder::new(&[(prompt, &vector_b)]); - let worker_b = RedisSemanticCache::with_connection(connection_b, embedder_b, config()) - .with_clock(|| 1700000000.5); - worker_b.set_cache(tag, value.clone(), &context()).unwrap(); + let worker_b = cache( + vec![ + info(INDEX, compatible_info(8)), + info_missing(&isolated), + create_index(&isolated, 4), + hset(&isolated, prompt, "key1", &vector_b, &entry), + search( + &isolated, + "key1", + &vector_b, + Ok(hit("key1", "0.0", encoded(&entry))), + ), + search( + &isolated, + "key1", + &vector_b, + Err(redis::RedisError::from(( + redis::ErrorKind::Extension, + "Vector dimension mismatch", + ))), + ), + ], + FakeEmbedder::new(&[(prompt, &vector_b)]), + ); + worker_b + .set_cache("key1", entry.clone(), &context()) + .unwrap(); assert_eq!( - worker_b.get_cache(tag, &context()).unwrap(), - Some(value.clone()) + worker_b.get_cache("key1", &context()).unwrap(), + Some(entry.clone()) ); let vector_c = vec![0.3f32; 16]; - let connection_c = MockRedisConnection::new([ - MockCmd::new(redis::cmd("FT.INFO").arg(INDEX), Ok(compatible_info(8))), - MockCmd::new(redis::cmd("FT.INFO").arg(&isolated), Ok(compatible_info(4))), - MockCmd::new(redis::cmd("FT.DROPINDEX").arg(&isolated), Ok("OK")), - MockCmd::new(create_index_command(&isolated, 16), Ok("OK")), - store_hash(&isolated, &vector_c), - ]) - .assert_all_commands_consumed(); - let (embedder_c, _) = FakeEmbedder::new(&[(prompt, &vector_c)]); - let worker_c = RedisSemanticCache::with_connection(connection_c, embedder_c, config()) - .with_clock(|| 1700000000.5); - worker_c.set_cache(tag, value.clone(), &context()).unwrap(); + let worker_c = cache( + vec![ + info(INDEX, compatible_info(8)), + info(&isolated, compatible_info(4)), + MockCmd::new(redis::cmd("FT.DROPINDEX").arg(&isolated), Ok("OK")), + create_index(&isolated, 16), + hset(&isolated, prompt, "key1", &vector_c, &entry), + ], + FakeEmbedder::new(&[(prompt, &vector_c)]), + ); + worker_c + .set_cache("key1", entry.clone(), &context()) + .unwrap(); assert_eq!( - worker_b.get_cache(tag, &context()).unwrap_err(), + worker_b.get_cache("key1", &context()).unwrap_err(), Error::Unavailable ); } -#[test] -fn live_shared_index_is_replaced_across_dimensions() { - let Ok(url) = std::env::var("LITELLM_REDIS_STACK_URL") else { +#[fixture] +fn redis_stack_url() -> Option { + std::env::var("LITELLM_REDIS_STACK_URL").ok() +} + +fn live_cache( + url: &str, + index_name: &str, + prompt: &str, + vector: Vec, +) -> RedisSemanticCache> { + RedisSemanticCache::new( + url, + FakeEmbedder::new(&[(prompt, vector.as_slice())]), + JsonCodec::::new(), + RedisSemanticConfig { + index_name: index_name.to_owned(), + similarity_threshold: 0.9, + }, + ) + .unwrap() +} + +#[rstest] +fn live_shared_index_is_replaced_across_dimensions(redis_stack_url: Option, entry: Value) { + let Some(url) = redis_stack_url else { return; }; // Pins parity with Python's `_isolated` + overwrite=True flow. let base = format!("rust_semantic_shared_{}", std::process::id()); let isolated = format!("{base}_isolated"); let prompt = "shared live prompt"; - let tag = "key1"; let context = || messages_context(vec![json!({"role": "user", "content": prompt})]); - let value = entry(); - let worker = |vector: Vec| { - let (embedder, _) = FakeEmbedder::new(&[(prompt, vector.as_slice())]); - RedisSemanticCache::new( - &url, - embedder, - RedisSemanticConfig { - index_name: base.clone(), - similarity_threshold: 0.9, - }, - ) - .unwrap() - }; - let worker_a = worker(vec![0.1f32; 8]); - worker_a.set_cache(tag, value.clone(), &context()).unwrap(); + let worker_a = live_cache(&url, &base, prompt, vec![0.1f32; 8]); + worker_a + .set_cache("key1", entry.clone(), &context()) + .unwrap(); - let worker_b = worker(vec![0.2f32; 4]); - worker_b.set_cache(tag, value.clone(), &context()).unwrap(); + let worker_b = live_cache(&url, &base, prompt, vec![0.2f32; 4]); + worker_b + .set_cache("key1", entry.clone(), &context()) + .unwrap(); assert_eq!( - worker_b.get_cache(tag, &context()).unwrap(), - Some(value.clone()) + worker_b.get_cache("key1", &context()).unwrap(), + Some(entry.clone()) ); - let worker_c = worker(vec![0.3f32; 16]); - worker_c.set_cache(tag, value.clone(), &context()).unwrap(); + let worker_c = live_cache(&url, &base, prompt, vec![0.3f32; 16]); + worker_c + .set_cache("key1", entry.clone(), &context()) + .unwrap(); assert_eq!( - worker_b.get_cache(tag, &context()).unwrap_err(), + worker_b.get_cache("key1", &context()).unwrap_err(), Error::Unavailable ); @@ -961,39 +746,29 @@ fn live_shared_index_is_replaced_across_dimensions() { } } -#[test] -fn live_store_lookup_and_ttl_against_redis_stack() { - let Ok(url) = std::env::var("LITELLM_REDIS_STACK_URL") else { +#[rstest] +fn live_store_lookup_and_ttl_against_redis_stack(redis_stack_url: Option, entry: Value) { + let Some(url) = redis_stack_url else { return; }; - let vector = vec![0.1f32, 0.2, 0.3, 0.4]; let prompt = "rust semantic cache live prompt"; - let tag = "live-key"; let index_name = format!("rust_semantic_test_{}", std::process::id()); - let (embedder, _) = FakeEmbedder::new(&[(prompt, &vector)]); - let cache = RedisSemanticCache::new( - &url, - embedder, - RedisSemanticConfig { - index_name: index_name.clone(), - similarity_threshold: 0.9, - }, - ) - .unwrap(); + let cache = live_cache(&url, &index_name, prompt, vec![0.1, 0.2, 0.3, 0.4]); let context = SemanticCacheContext { ttl: Some(Duration::from_secs(120)), ..messages_context(vec![json!({"role": "user", "content": prompt})]) }; - let value = entry(); - cache.set_cache(tag, value.clone(), &context).unwrap(); - assert_eq!(cache.get_cache(tag, &context).unwrap(), Some(value)); + cache + .set_cache("live-key", entry.clone(), &context) + .unwrap(); + assert_eq!(cache.get_cache("live-key", &context).unwrap(), Some(entry)); assert_eq!(cache.get_cache("other-key", &context).unwrap(), None); let mut connection = redis::Client::open(url).unwrap().get_connection().unwrap(); let ttl: i64 = redis::Commands::ttl( &mut connection, - format!("{index_name}:{}", entry_id(prompt, tag)), + format!("{index_name}:{}", entry_id(prompt, "live-key")), ) .unwrap(); assert!( diff --git a/litellm-rust/crates/cache-redis-semantic/tests/contract.rs b/litellm-rust/crates/cache-redis-semantic/tests/contract.rs new file mode 100644 index 00000000000..fb3cda9616e --- /dev/null +++ b/litellm-rust/crates/cache-redis-semantic/tests/contract.rs @@ -0,0 +1,63 @@ +mod support; + +use litellm_cache::{JsonCodec, SemanticCacheContext, semantic::PreparedEmbedding}; +use litellm_cache_redis_semantic::{DEFAULT_INDEX_NAME, RedisSemanticCache, RedisSemanticConfig}; +use litellm_cache_testing as contract; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; +use support::FakeSearch; + +type Cache = RedisSemanticCache, FakeSearch>; + +const PREFIX: &str = "contract:"; + +#[fixture] +fn cache() -> Cache { + RedisSemanticCache::with_connection( + FakeSearch::default(), + PreparedEmbedding(vec![0.6, 0.8]), + JsonCodec::new(), + RedisSemanticConfig { + index_name: DEFAULT_INDEX_NAME.into(), + similarity_threshold: 0.9, + }, + ) +} + +#[fixture] +fn context() -> SemanticCacheContext { + SemanticCacheContext { + messages: Some(json!([{"role": "user", "content": "contract prompt"}])), + ..Default::default() + } +} + +#[rstest] +#[tokio::test] +async fn hit_and_miss(cache: Cache, context: SemanticCacheContext) { + contract::hit_and_miss(&cache, context, PREFIX, json!({"answer": 42})).await; +} + +#[rstest] +#[tokio::test] +async fn sync_async_equivalence(cache: Cache, context: SemanticCacheContext) { + contract::sync_async_equivalence(&cache, context, PREFIX, json!("first"), json!([2])).await; +} + +#[rstest] +#[tokio::test] +async fn overwrite_replaces(cache: Cache, context: SemanticCacheContext) { + contract::overwrite_replaces(&cache, context, PREFIX, json!(1), json!({"b": 2})).await; +} + +#[rstest] +#[tokio::test] +async fn pipeline_writes_every_entry(cache: Cache, context: SemanticCacheContext) { + contract::pipeline_writes_every_entry( + &cache, + context, + PREFIX, + vec![json!("a"), json!(2), json!({"c": true})], + ) + .await; +} diff --git a/litellm-rust/crates/cache-redis-semantic/tests/support/mod.rs b/litellm-rust/crates/cache-redis-semantic/tests/support/mod.rs new file mode 100644 index 00000000000..0f5c2985c36 --- /dev/null +++ b/litellm-rust/crates/cache-redis-semantic/tests/support/mod.rs @@ -0,0 +1,299 @@ +#![allow(dead_code)] + +use std::{ + collections::{BTreeMap, HashMap}, + sync::{Arc, Mutex}, +}; + +use litellm_cache::{Error, semantic::Embedder}; +use serde_json::Value; + +pub type EmbedCalls = Arc)>>>; + +/// Embeds known prompts to fixed vectors, anything else to `[0.1, 0.2, 0.3]`, and records every +/// prompt with its metadata. +pub struct FakeEmbedder { + vectors: HashMap>, + pub calls: EmbedCalls, +} + +impl FakeEmbedder { + pub fn new(vectors: &[(&str, &[f32])]) -> Self { + Self { + vectors: vectors + .iter() + .map(|(prompt, vector)| ((*prompt).to_owned(), vector.to_vec())) + .collect(), + calls: EmbedCalls::default(), + } + } +} + +impl Embedder for FakeEmbedder { + fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { + self.calls + .lock() + .unwrap() + .push((prompt.to_owned(), metadata.cloned())); + Ok(self + .vectors + .get(prompt) + .cloned() + .unwrap_or_else(|| vec![0.1, 0.2, 0.3])) + } + + async fn async_embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { + self.embed(prompt, metadata) + } +} + +struct FakeIndex { + prefix: Vec, + dims: usize, + vector_field: String, +} + +#[derive(Default)] +struct SearchState { + indexes: HashMap, + hashes: BTreeMap, BTreeMap>>, +} + +/// An in-memory Redis Stack speaking the `FT.*`, `HSET` and `EXPIRE` subset the semantic cache +/// sends, with exact cosine KNN over the hashes under an index prefix. +#[derive(Clone, Default)] +pub struct FakeSearch { + state: Arc>, +} + +impl FakeSearch { + fn run(&self, args: Vec>) -> redis::RedisResult { + let mut state = self.state.lock().unwrap(); + let text = |index: usize| String::from_utf8_lossy(&args[index]).into_owned(); + match text(0).to_uppercase().as_str() { + "FT.CREATE" => { + let name = text(1); + if state.indexes.contains_key(&name) { + return Err(error("Index already exists")); + } + let position = |token: &str| args.iter().position(|arg| arg == token.as_bytes()); + let prefix = args[position("PREFIX").unwrap() + 2].clone(); + let dims = text(position("DIM").unwrap() + 1).parse().unwrap(); + let vector_field = text(position("VECTOR").unwrap() - 1); + state.indexes.insert( + name, + FakeIndex { + prefix, + dims, + vector_field, + }, + ); + Ok(redis::Value::Okay) + } + "FT.INFO" => { + let index = state + .indexes + .get(&text(1)) + .ok_or_else(|| error("Unknown index name"))?; + Ok(index_info(index)) + } + "FT.DROPINDEX" => { + state.indexes.remove(&text(1)); + Ok(redis::Value::Okay) + } + "HSET" => { + let hash = state.hashes.entry(args[1].clone()).or_default(); + for pair in args[2..].chunks(2) { + hash.insert( + String::from_utf8_lossy(&pair[0]).into_owned(), + pair[1].clone(), + ); + } + Ok(redis::Value::Int(((args.len() - 2) / 2) as i64)) + } + "EXPIRE" => Ok(redis::Value::Int(i64::from( + state.hashes.contains_key(&args[1]), + ))), + "FT.SEARCH" => { + let index = state + .indexes + .get(&text(1)) + .ok_or_else(|| error("no such index"))?; + let query = text(2); + let tag = query_tag(&query); + let params = args.iter().position(|arg| arg == b"PARAMS").unwrap(); + let vector = floats(&args[params + 3]); + let best = state + .hashes + .iter() + .filter(|(key, _)| key.starts_with(&index.prefix)) + .filter(|(_, fields)| { + fields.get("litellm_cache_key").map(Vec::as_slice) == Some(tag.as_bytes()) + }) + .filter_map(|(key, fields)| { + let stored = floats(fields.get(&index.vector_field)?); + (stored.len() == index.dims) + .then(|| (key, fields, 1.0 - cosine(&vector, &stored))) + }) + .min_by(|left, right| left.2.total_cmp(&right.2)); + let Some((key, fields, distance)) = best else { + return Ok(redis::Value::Array(vec![redis::Value::Int(0)])); + }; + let mut reply = fields + .iter() + .filter(|(name, _)| **name != index.vector_field) + .flat_map(|(name, value)| [bulk(name.as_bytes()), bulk(value)]) + .collect::>(); + reply.extend([ + bulk(b"vector_distance"), + bulk(distance.to_string().as_bytes()), + ]); + Ok(redis::Value::Array(vec![ + redis::Value::Int(1), + bulk(key), + redis::Value::Array(reply), + ])) + } + "PING" => Ok(redis::Value::SimpleString("PONG".into())), + _ => Err(error("unsupported command")), + } + } +} + +impl redis::ConnectionLike for FakeSearch { + fn req_packed_command(&mut self, command: &[u8]) -> redis::RedisResult { + let mut commands = parse_commands(command); + self.run(commands.remove(0)) + } + + fn req_packed_commands( + &mut self, + commands: &[u8], + offset: usize, + count: usize, + ) -> redis::RedisResult> { + let replies = parse_commands(commands) + .into_iter() + .map(|args| self.run(args)) + .collect::>>()?; + Ok(replies.into_iter().skip(offset).take(count).collect()) + } + + fn get_db(&self) -> i64 { + 0 + } + + fn check_connection(&mut self) -> bool { + true + } + + fn is_open(&self) -> bool { + true + } +} + +fn error(message: &'static str) -> redis::RedisError { + redis::RedisError::from((redis::ErrorKind::Extension, message)) +} + +fn bulk(bytes: &[u8]) -> redis::Value { + redis::Value::BulkString(bytes.to_vec()) +} + +fn index_info(index: &FakeIndex) -> redis::Value { + let attribute = |name: &str, field_type: &str| { + redis::Value::Array(vec![ + bulk(b"identifier"), + bulk(name.as_bytes()), + bulk(b"type"), + bulk(field_type.as_bytes()), + ]) + }; + redis::Value::Array(vec![ + bulk(b"attributes"), + redis::Value::Array(vec![ + attribute("prompt", "TEXT"), + attribute("response", "TEXT"), + attribute("inserted_at", "NUMERIC"), + attribute("updated_at", "NUMERIC"), + attribute("litellm_cache_key", "TAG"), + redis::Value::Array(vec![ + bulk(b"identifier"), + bulk(index.vector_field.as_bytes()), + bulk(b"type"), + bulk(b"VECTOR"), + bulk(b"dim"), + redis::Value::Int(index.dims as i64), + bulk(b"data_type"), + bulk(b"FLOAT32"), + bulk(b"distance_metric"), + bulk(b"COSINE"), + ]), + ]), + ]) +} + +/// The tag inside `@litellm_cache_key:{...}`, with query escapes removed. +fn query_tag(query: &str) -> String { + let start = query.find("@litellm_cache_key:{").unwrap() + "@litellm_cache_key:{".len(); + let mut tag = String::new(); + let mut characters = query[start..].chars(); + while let Some(character) = characters.next() { + match character { + '\\' => tag.extend(characters.next()), + '}' => break, + character => tag.push(character), + } + } + tag +} + +fn floats(bytes: &[u8]) -> Vec { + bytes + .as_chunks::<4>() + .0 + .iter() + .map(|chunk| f32::from_le_bytes(*chunk)) + .collect() +} + +fn cosine(left: &[f32], right: &[f32]) -> f64 { + let dot = left + .iter() + .zip(right) + .map(|(left, right)| f64::from(*left) * f64::from(*right)) + .sum::(); + let norm = |vector: &[f32]| { + vector + .iter() + .map(|value| f64::from(*value).powi(2)) + .sum::() + .sqrt() + }; + dot / (norm(left) * norm(right)) +} + +/// Splits a packed RESP request into each command's arguments. +fn parse_commands(mut bytes: &[u8]) -> Vec>> { + let line = |bytes: &mut &[u8]| { + let end = bytes + .windows(2) + .position(|window| window == b"\r\n") + .unwrap(); + let text = String::from_utf8(bytes[1..end].to_vec()).unwrap(); + *bytes = &bytes[end + 2..]; + text.parse::().unwrap() + }; + let mut commands = Vec::new(); + while !bytes.is_empty() { + let count = line(&mut bytes); + let mut args = Vec::with_capacity(count); + for _ in 0..count { + let length = line(&mut bytes); + args.push(bytes[..length].to_vec()); + bytes = &bytes[length + 2..]; + } + commands.push(args); + } + commands +} diff --git a/litellm-rust/crates/cache-redis/Cargo.toml b/litellm-rust/crates/cache-redis/Cargo.toml index ea937098698..a234286a338 100644 --- a/litellm-rust/crates/cache-redis/Cargo.toml +++ b/litellm-rust/crates/cache-redis/Cargo.toml @@ -12,5 +12,7 @@ r2d2 = "0.8.10" tokio.workspace = true [dev-dependencies] +litellm-cache-testing.workspace = true redis-test = "1.0.4" +rstest.workspace = true serde_json.workspace = true diff --git a/litellm-rust/crates/cache-redis/src/cache.rs b/litellm-rust/crates/cache-redis/src/cache.rs index 24399c9b2f9..ebcf0b6916b 100644 --- a/litellm-rust/crates/cache-redis/src/cache.rs +++ b/litellm-rust/crates/cache-redis/src/cache.rs @@ -1,110 +1,25 @@ use std::{ - sync::{Arc, Mutex}, + sync::{Arc, OnceLock}, time::Duration, }; -use litellm_cache::{ - BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionResult, CacheConnectionStatus, - ClaimCache, CounterCache, DeleteCache, Error, ExactCacheContext, FlushCache, -}; -use redis::Commands; +use litellm_cache::{BatchEntry, CacheCodec, Error}; -use crate::topology::RedisTopology; - -mod connection; -mod operations; - -pub use connection::ConnectionRef; -use connection::{ClusterConnectionManager, ConnectionManager}; - -pub use operations::{ - RedisArg, RedisLpopOperation, RedisLpopResult, RedisRpushOperation, RedisScript, +use crate::{ + connection::{ConnectionRef, Connections}, + topology::RedisTopology, }; const DEFAULT_TTL: Duration = Duration::from_secs(600); -const REDIS_TIMEOUT: Duration = Duration::from_secs(5); -const REDIS_POOL_SIZE: u32 = 16; - -const INCREMENT_SCRIPT: &str = concat!( - "local value = redis.call('INCRBYFLOAT', KEYS[1], ARGV[1]); ", - "if redis.call('TTL', KEYS[1]) == -1 then ", - "redis.call('EXPIRE', KEYS[1], ARGV[2]); end; return value" -); - -const CLAIM_SCRIPT: &str = concat!( - "local current = redis.call('GET', KEYS[1]); ", - "if ARGV[1] == '' then if current ~= false and current ~= '' then return 0; end; ", - "elseif current ~= ARGV[1] then return 0; end; ", - "if ARGV[3] ~= '' then redis.call('SET', KEYS[1], ARGV[3], 'EX', ARGV[2]); ", - "elseif ARGV[4] == '1' then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; return 1" -); -const CLAIM_ATTEMPTS: usize = 8; - -#[allow(private_interfaces)] -pub enum Connections { - Pool(r2d2::Pool), - Cluster(r2d2::Pool), - Fixed(Mutex), -} - -impl Connections -where - C: redis::ConnectionLike + Send + 'static, -{ - pub fn execute( - &self, - operation: impl FnOnce(&mut ConnectionRef<'_>) -> Result, - ) -> Result { - match self { - Self::Pool(pool) => { - let mut pooled = pool.get().map_err(|_| Error::Unavailable)?; - let result = operation(&mut ConnectionRef::Node(&mut pooled.connection)); - pooled.failed = matches!(result, Err(Error::Unavailable)); - result - } - Self::Cluster(pool) => { - let mut pooled = pool.get().map_err(|_| Error::Unavailable)?; - let result = operation(&mut ConnectionRef::Cluster(&mut pooled.connection)); - pooled.failed = matches!(result, Err(Error::Unavailable)); - result - } - Self::Fixed(connection) => { - let mut connection = connection.lock().map_err(|_| Error::Unavailable)?; - operation(&mut ConnectionRef::Node(&mut *connection)) - } - } - } - - pub async fn run_blocking(connections: Arc, operation: F) -> Result - where - T: Send + 'static, - F: FnOnce(&mut ConnectionRef<'_>) -> Result + Send + 'static, - { - tokio::task::spawn_blocking(move || connections.execute(operation)) - .await - .map_err(|_| Error::Unavailable)? - } - - pub fn fixed(connection: C) -> Self { - Self::Fixed(Mutex::new(connection)) - } - - pub fn open(url: &str, topology: &RedisTopology) -> Result { - match topology { - RedisTopology::Standalone => Ok(Self::Pool(pool(ConnectionManager::open(url)?)?)), - RedisTopology::Cluster { startup_nodes } => Ok(Self::Cluster(pool( - ClusterConnectionManager::open(url, startup_nodes)?, - )?)), - } - } -} pub struct RedisCache { - connections: Arc>, - default_ttl: Duration, - codec: S, - namespace: Option, - topology: RedisTopology, + pub(crate) connections: Arc>, + pub(crate) default_ttl: Duration, + pub(crate) codec: S, + pub(crate) namespace: Option, + pub(crate) topology: RedisTopology, + /// The server's major version, read from `INFO` once, like Python's `redis_version`. + pub(crate) major_version: Arc>, } impl RedisCache { @@ -125,20 +40,11 @@ impl RedisCache { codec, namespace: None, topology: topology.clone(), + major_version: Arc::default(), }) } } -fn pool(manager: M) -> Result, Error> { - r2d2::Pool::builder() - .max_size(REDIS_POOL_SIZE) - .min_idle(Some(0)) - .connection_timeout(REDIS_TIMEOUT) - .test_on_check_out(false) - .build(manager) - .map_err(|_| Error::Unavailable) -} - impl RedisCache where S: CacheCodec, @@ -151,6 +57,7 @@ where codec, namespace: None, topology: RedisTopology::Standalone, + major_version: Arc::default(), } } @@ -169,11 +76,52 @@ where &self.topology } - fn namespaced_key(&self, key: &str) -> String { + pub(crate) fn namespaced_key(&self, key: &str) -> String { namespaced_key(self.namespace.as_deref(), key) } - fn namespaced_pattern(&self) -> Result { + pub(crate) fn namespaced_keys(&self, keys: &[String]) -> Vec { + keys.iter().map(|key| self.namespaced_key(key)).collect() + } + + /// Whole seconds for `ttl`, falling back to the default TTL like Python's `get_ttl`. + pub(crate) fn ttl_or_default(&self, ttl: Option) -> u64 { + ttl_seconds(ttl.unwrap_or(self.default_ttl)) + } + + pub(crate) fn execute( + &self, + operation: impl FnOnce(&mut ConnectionRef<'_>) -> Result, + ) -> Result { + self.connections.execute(operation) + } + + /// `_parse_redis_major_version`: the major version from `INFO`, or + /// `DEFAULT_REDIS_MAJOR_VERSION` when `INFO` fails or its version does not parse. The first + /// answer is kept, as Python reads `redis_version` once at construction. + pub(crate) async fn major_version(&self) -> u32 { + if let Some(version) = self.major_version.get() { + return *version; + } + let info = self + .run(|connection| connection.node_text(&redis::cmd("INFO"))) + .await; + let version = info + .ok() + .and_then(|info| parse_major_version(&info)) + .unwrap_or_else(default_major_version); + *self.major_version.get_or_init(|| version) + } + + pub(crate) async fn run(&self, operation: F) -> Result + where + T: Send + 'static, + F: FnOnce(&mut ConnectionRef<'_>) -> Result + Send + 'static, + { + Connections::run_blocking(Arc::clone(&self.connections), operation).await + } + + pub(crate) fn namespaced_pattern(&self) -> Result { let namespace = self.namespace.as_ref().ok_or(Error::UnscopedFlush)?; let escaped: String = namespace .chars() @@ -188,18 +136,7 @@ where Ok(format!("{escaped}:*")) } - fn flush_matching(connection: &mut ConnectionRef<'_>, pattern: &str) -> Result<(), Error> { - connection.scan(pattern, 1000, |connection, keys| { - if !keys.is_empty() { - connection - .del::<_, usize>(keys) - .map_err(|_| Error::Unavailable)?; - } - Ok(true) - }) - } - - fn decode_response(&self, value: redis::Value) -> Result, Error> { + pub(crate) fn decode_response(&self, value: redis::Value) -> Result, Error> { match value { redis::Value::Nil => Ok(None), redis::Value::BulkString(bytes) => self.codec.decode(&bytes).map(Some), @@ -208,7 +145,10 @@ where } } - fn decode_batch_response(&self, value: redis::Value) -> Result, Error> { + pub(crate) fn decode_batch_response( + &self, + value: redis::Value, + ) -> Result, Error> { match self.decode_response(value) { Ok(Some(value)) => Ok(BatchEntry::Hit(value)), Ok(None) => Ok(BatchEntry::Miss), @@ -216,15 +156,9 @@ where Err(error) => Err(error), } } - - fn ttl_seconds(ttl: Duration) -> u64 { - ttl.as_secs() - .saturating_add(u64::from(ttl.subsec_nanos() > 0)) - .max(1) - } } -fn namespaced_key(namespace: Option<&str>, key: &str) -> String { +pub(crate) fn namespaced_key(namespace: Option<&str>, key: &str) -> String { match namespace { Some(namespace) if !key.starts_with(&format!("{namespace}:")) => { format!("{namespace}:{key}") @@ -233,469 +167,26 @@ fn namespaced_key(namespace: Option<&str>, key: &str) -> String { } } -impl BaseCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - type Value = S::Value; - type Context = ExactCacheContext; +pub(crate) fn ttl_seconds(ttl: Duration) -> u64 { + ttl.as_secs() + .saturating_add(u64::from(ttl.subsec_nanos() > 0)) + .max(1) +} - fn get_ttl(&self, context: &Self::Context) -> Option { - context.ttl.or(Some(self.default_ttl)) - } - - fn set_cache( - &self, - key: &str, - value: Self::Value, - context: &ExactCacheContext, - ) -> Result<(), Error> { - let payload = self.codec.encode(&value)?; - let ttl = Self::ttl_seconds(self.get_ttl(context).unwrap_or(self.default_ttl)); - let key = self.namespaced_key(key); - self.connections.execute(|connection| { - connection - .set_ex::<_, _, ()>(key, payload, ttl) - .map_err(|_| Error::Unavailable) - }) - } - - fn get_cache(&self, key: &str, _: &ExactCacheContext) -> Result, Error> { - let key = self.namespaced_key(key); - let value = self.connections.execute(|connection| { - connection - .get::<_, redis::Value>(key) - .map_err(|_| Error::Unavailable) - })?; - self.decode_response(value) - } - - async fn async_set_cache( - &self, - key: &str, - value: Self::Value, - context: ExactCacheContext, - ) -> Result<(), Error> { - let payload = self.codec.encode(&value)?; - let key = self.namespaced_key(key); - let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl)); - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - connection - .set_ex::<_, _, ()>(key, payload, ttl) - .map_err(|_| Error::Unavailable) - }) - .await - } - - async fn async_get_cache( - &self, - key: &str, - _: &ExactCacheContext, - ) -> Result, Error> { - let key = self.namespaced_key(key); - let value = Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - connection - .get::<_, redis::Value>(key) - .map_err(|_| Error::Unavailable) - }) - .await?; - self.decode_response(value) - } - - async fn async_set_cache_pipeline( - &self, - cache_list: Vec<(String, Self::Value)>, - context: ExactCacheContext, - ) -> Result<(), Error> { - let entries = cache_list - .into_iter() - .map(|(key, value)| { - self.codec - .encode(&value) - .map(|payload| (self.namespaced_key(&key), payload)) - }) - .collect::, _>>()?; - let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl)); - if entries.is_empty() { - return Ok(()); - } - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - let commands = entries - .into_iter() - .map(|(key, payload)| { - let mut command = redis::cmd("SETEX"); - command.arg(key).arg(ttl).arg(payload); - command - }) - .collect(); - connection.pipeline(commands).map(drop) - }) - .await - } - - async fn disconnect(&self) -> Result<(), Error> { - Ok(()) - } - - async fn test_connection(&self) -> Result { - match Connections::run_blocking(Arc::clone(&self.connections), |connection| { - Ok(match connection.ping() { - Ok(_) => CacheConnectionResult { - status: CacheConnectionStatus::Success, - message: "Redis cache connection test successful".into(), - error: None, - }, - Err(error) => CacheConnectionResult { - status: CacheConnectionStatus::Failed, - message: format!("Redis connection failed: {error}"), - error: Some(error.to_string()), - }, - }) - }) - .await - { - Ok(result) => Ok(result), - Err(error) => Ok(CacheConnectionResult { - status: CacheConnectionStatus::Failed, - message: format!("Redis connection failed: {error}"), - error: Some(error.to_string()), - }), - } +fn parse_major_version(info: &str) -> Option { + let version = info + .lines() + .find_map(|line| line.trim().strip_prefix("redis_version:"))? + .trim(); + match version.split_once('.') { + Some((major, _)) => major.parse().ok(), + None => version.parse::().ok().map(|major| major as u32), } } -impl BatchCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - fn batch_get_cache( - &self, - keys: &[String], - _: &ExactCacheContext, - ) -> Result>, Error> { - let keys = keys - .iter() - .map(|key| self.namespaced_key(key)) - .collect::>(); - let values = self.connections.execute(|connection| { - redis::cmd("MGET") - .arg(keys) - .query::>(connection) - .map_err(|_| Error::Unavailable) - })?; - values - .into_iter() - .map(|value| self.decode_batch_response(value)) - .collect() - } - - async fn async_batch_get_cache( - &self, - keys: Vec, - _: ExactCacheContext, - ) -> Result>, Error> { - let keys = keys - .iter() - .map(|key| self.namespaced_key(key)) - .collect::>(); - let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - redis::cmd("MGET") - .arg(keys) - .query::>(connection) - .map_err(|_| Error::Unavailable) - }) - .await?; - values - .into_iter() - .map(|value| self.decode_batch_response(value)) - .collect() - } -} - -impl DeleteCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - fn delete_cache(&self, key: &str) -> Result<(), Error> { - let key = self.namespaced_key(key); - self.connections - .execute(|connection| connection.del::<_, ()>(key).map_err(|_| Error::Unavailable)) - } - - async fn async_delete_cache(&self, key: &str) -> Result<(), Error> { - let key = self.namespaced_key(key); - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - connection.del::<_, ()>(key).map_err(|_| Error::Unavailable) - }) - .await - } -} - -impl FlushCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - fn flush_cache(&self) -> Result<(), Error> { - let pattern = self.namespaced_pattern()?; - self.connections - .execute(|connection| Self::flush_matching(connection, &pattern)) - } - - async fn async_flush_cache(&self) -> Result<(), Error> { - let pattern = self.namespaced_pattern()?; - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - Self::flush_matching(connection, &pattern) - }) - .await - } -} - -impl CounterCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - fn increment_cache( - &self, - key: &str, - amount: f64, - context: ExactCacheContext, - ) -> Result { - let key = self.namespaced_key(key); - let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl)); - self.connections - .execute(|connection| increment(connection, key, amount, ttl)) - } - - async fn async_increment( - &self, - key: &str, - amount: f64, - context: ExactCacheContext, - ) -> Result { - let key = self.namespaced_key(key); - let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl)); - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - increment(connection, key, amount, ttl) - }) - .await - } -} - -fn increment( - connection: &mut ConnectionRef<'_>, - key: String, - amount: f64, - ttl: u64, -) -> Result { - redis::cmd("EVAL") - .arg(INCREMENT_SCRIPT) - .arg(1) - .arg(key) - .arg(amount) - .arg(ttl) - .query(connection) - .map_err(|_| Error::Unavailable) -} - -fn stored_bytes(value: redis::Value) -> Result>, Error> { - match value { - redis::Value::Nil => Ok(None), - redis::Value::BulkString(bytes) => Ok(Some(bytes)), - redis::Value::SimpleString(text) => Ok(Some(text.into_bytes())), - _ => Err(Error::InvalidEntry), - } -} - -/// Eligibility is decided on decoded values, so a pin written by another encoder (Python's -/// `json.dumps` spacing or key order) still matches. The write is a compare-and-set on the -/// bytes that decision was made on, retried when another claimant wins the race. -fn claim( - connection: &mut ConnectionRef<'_>, - codec: &S, - key: &str, - candidate: S::Value, - eligible: &[S::Value], - ttl: u64, -) -> Result -where - S::Value: PartialEq, -{ - let payload = codec.encode(&candidate)?; - if payload.is_empty() { - return Err(Error::InvalidEntry); - } - for _ in 0..CLAIM_ATTEMPTS { - let current = stored_bytes( - connection - .get::<_, redis::Value>(key) - .map_err(|_| Error::Unavailable)?, - )? - .filter(|bytes| !bytes.is_empty()); - let existing = current - .as_deref() - .and_then(|bytes| codec.decode(bytes).ok()) - .filter(|existing| eligible.is_empty() || eligible.contains(existing)); - let refresh = existing - .as_ref() - .is_some_and(|existing| !eligible.is_empty() || *existing == candidate); - let write: &[u8] = if existing.is_some() { b"" } else { &payload }; - let applied = redis::cmd("EVAL") - .arg(CLAIM_SCRIPT) - .arg(1) - .arg(key) - .arg(current.as_deref().unwrap_or_default()) - .arg(ttl) - .arg(write) - .arg(u8::from(refresh)) - .query::(connection) - .map_err(|_| Error::Unavailable)?; - if applied { - return Ok(existing.unwrap_or(candidate)); - } - } - Err(Error::Unavailable) -} - -impl ClaimCache for RedisCache -where - S: CacheCodec + Clone + 'static, - S::Value: PartialEq, - C: redis::ConnectionLike + Send + 'static, -{ - fn claim_cache( - &self, - key: &str, - candidate: S::Value, - eligible: &[S::Value], - context: ExactCacheContext, - ) -> Result { - let key = self.namespaced_key(key); - let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl)); - self.connections - .execute(|connection| claim(connection, &self.codec, &key, candidate, eligible, ttl)) - } - - async fn async_claim_cache( - &self, - key: &str, - candidate: S::Value, - eligible: Vec, - context: ExactCacheContext, - ) -> Result { - let key = self.namespaced_key(key); - let ttl = Self::ttl_seconds(self.get_ttl(&context).unwrap_or(self.default_ttl)); - let codec = self.codec.clone(); - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - claim(connection, &codec, &key, candidate, &eligible, ttl) - }) - .await - } -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use litellm_cache::{ - BaseCache, CacheCodec, DeleteCache, ExactCacheContext, FlushCache, JsonCodec, - }; - use redis_test::{MockCmd, MockRedisConnection}; - use serde_json::json; - - use super::RedisCache; - - fn entry() -> serde_json::Value { - json!({"deployment": "model-a", "cooldown_seconds": 30}) - } - - #[test] - fn ttl_seconds_rounds_up_and_keeps_expiration_positive() { - assert_eq!( - RedisCache::>::ttl_seconds(Duration::ZERO), - 1 - ); - assert_eq!( - RedisCache::>::ttl_seconds(Duration::from_millis(1500)), - 2 - ); - assert_eq!( - RedisCache::>::ttl_seconds(Duration::from_secs(15)), - 15 - ); - } - - #[test] - fn redis_commands_round_trip_entries_and_delete_only_namespaced_keys() { - let value = entry(); - let payload = JsonCodec::::new() - .encode(&value) - .unwrap(); - let connection = MockRedisConnection::new([ - MockCmd::new( - redis::cmd("SETEX") - .arg("litellm-cache:key") - .arg(600) - .arg(payload.clone()), - Ok("OK"), - ), - MockCmd::new(redis::cmd("GET").arg("litellm-cache:key"), Ok(payload)), - MockCmd::new(redis::cmd("DEL").arg("litellm-cache:key"), Ok(1u32)), - ]) - .assert_all_commands_consumed(); - let cache = - RedisCache::with_connection(connection, None, JsonCodec::::new()) - .with_namespace(Some("litellm-cache".into())); - - cache - .set_cache("key", value.clone(), &ExactCacheContext::default()) - .unwrap(); - assert_eq!( - cache - .get_cache("key", &ExactCacheContext::default()) - .unwrap(), - Some(value) - ); - cache.delete_cache("key").unwrap(); - } - - #[test] - fn flush_scans_and_deletes_only_cache_keys() { - let connection = MockRedisConnection::new([ - MockCmd::new( - redis::cmd("SCAN") - .cursor_arg(0) - .arg("MATCH") - .arg("litellm-cache:*") - .arg("COUNT") - .arg(1000), - Ok(redis_test::redis_value!(["0", ["litellm-cache:key"]])), - ), - MockCmd::new(redis::cmd("DEL").arg("litellm-cache:key"), Ok(1u32)), - ]) - .assert_all_commands_consumed(); - let cache = - RedisCache::with_connection(connection, None, JsonCodec::::new()) - .with_namespace(Some("litellm-cache".into())); - - cache.flush_cache().unwrap(); - } - - #[tokio::test] - async fn test_connection_runs_ping_off_executor() { - let connection = MockRedisConnection::new([MockCmd::new(redis::cmd("PING"), Ok("PONG"))]) - .assert_all_commands_consumed(); - let cache = - RedisCache::with_connection(connection, None, JsonCodec::::new()) - .with_namespace(Some("litellm-cache".into())); - - assert_eq!( - cache.test_connection().await.unwrap().status, - litellm_cache::CacheConnectionStatus::Success - ); - } +fn default_major_version() -> u32 { + std::env::var("DEFAULT_REDIS_MAJOR_VERSION") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(7) } diff --git a/litellm-rust/crates/cache-redis/src/cache/operations.rs b/litellm-rust/crates/cache-redis/src/cache/operations.rs deleted file mode 100644 index 4345ee879b3..00000000000 --- a/litellm-rust/crates/cache-redis/src/cache/operations.rs +++ /dev/null @@ -1,632 +0,0 @@ -use std::{sync::Arc, time::Duration}; - -use litellm_cache::{ - CacheCodec, CacheScript, ClientInfoCache, Error, IncrementOperation, QueueCache, ScanCache, - ScriptCache, SetCache, TtlCache, -}; -use redis::Commands; - -use super::{ConnectionRef, Connections, RedisCache, namespaced_key}; - -const INCREMENT_WITH_FLOOR_SCRIPT: &str = concat!( - "local count = redis.call('INCRBY', KEYS[1], ARGV[1]); ", - "if count < 0 then count = redis.call('INCRBY', KEYS[1], -count); end; ", - "if redis.call('TTL', KEYS[1]) < 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; ", - "return count" -); -const SET_MAX_SCRIPT: &str = concat!( - "local current = redis.call('GET', KEYS[1]); ", - "if current == false or tonumber(current) < tonumber(ARGV[1]) then ", - "redis.call('SET', KEYS[1], ARGV[1]); ", - "if tonumber(ARGV[2]) > 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; ", - "return ARGV[1]; end; return current" -); - -#[derive(Clone, Debug, PartialEq)] -pub enum RedisArg { - Bytes(Vec), - Integer(i64), - Float(f64), -} - -impl From<&str> for RedisArg { - fn from(value: &str) -> Self { - Self::Bytes(value.as_bytes().to_vec()) - } -} - -impl From for RedisArg { - fn from(value: String) -> Self { - Self::Bytes(value.into_bytes()) - } -} - -impl From> for RedisArg { - fn from(value: Vec) -> Self { - Self::Bytes(value) - } -} - -impl From for RedisArg { - fn from(value: i64) -> Self { - Self::Integer(value) - } -} - -impl From for RedisArg { - fn from(value: f64) -> Self { - Self::Float(value) - } -} - -impl redis::ToRedisArgs for RedisArg { - fn write_redis_args(&self, out: &mut W) - where - W: ?Sized + redis::RedisWrite, - { - match self { - Self::Bytes(value) => value.write_redis_args(out), - Self::Integer(value) => value.write_redis_args(out), - Self::Float(value) => value.write_redis_args(out), - } - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct RedisRpushOperation { - pub key: String, - pub values: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RedisLpopOperation { - pub key: String, - pub count: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum RedisLpopResult { - Missing, - Value(Vec), - Values(Vec>), -} - -pub struct RedisScript { - connections: Arc>, - namespace: Option, - source: String, -} - -impl CacheScript for RedisScript -where - C: redis::ConnectionLike + Send + 'static, -{ - type Argument = RedisArg; - type Output = redis::Value; - - async fn invoke( - &self, - keys: Vec, - arguments: Vec, - ) -> Result { - let keys = keys - .into_iter() - .map(|key| namespaced_key(self.namespace.as_deref(), &key)) - .collect::>(); - let connections = Arc::clone(&self.connections); - let source = self.source.clone(); - tokio::task::spawn_blocking(move || { - connections.execute(|connection| { - redis::cmd("EVAL") - .arg(source) - .arg(keys.len()) - .arg(keys) - .arg(arguments) - .query(connection) - .map_err(|_| Error::Unavailable) - }) - }) - .await - .map_err(|_| Error::Unavailable)? - } -} - -impl RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - pub async fn delete_cache_keys(&self, keys: Vec) -> Result { - if keys.is_empty() { - return Ok(0); - } - let keys = keys - .into_iter() - .map(|key| self.namespaced_key(&key)) - .collect::>(); - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - connection.del(keys).map_err(|_| Error::Unavailable) - }) - .await - } - - pub fn batch_get_counts(&self, keys: &[String]) -> Result>, Error> { - let keys = keys - .iter() - .map(|key| self.namespaced_key(key)) - .collect::>(); - let values = self.connections.execute(|connection| { - redis::cmd("MGET") - .arg(keys) - .query::>(connection) - .map_err(|_| Error::Unavailable) - })?; - values.into_iter().map(count).collect() - } - - pub async fn async_batch_get_counts( - &self, - keys: Vec, - ) -> Result>, Error> { - let keys = keys - .iter() - .map(|key| self.namespaced_key(key)) - .collect::>(); - let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - redis::cmd("MGET") - .arg(keys) - .query::>(connection) - .map_err(|_| Error::Unavailable) - }) - .await?; - values.into_iter().map(count).collect() - } - - pub fn sync_ping(&self) -> Result { - self.connections - .execute(|connection| connection.ping().map_err(|_| Error::Unavailable)) - } - - pub async fn ping(&self) -> Result { - Connections::run_blocking(Arc::clone(&self.connections), |connection| { - connection.ping().map_err(|_| Error::Unavailable) - }) - .await - } - - pub async fn async_get_ttl(&self, key: &str) -> Result, Error> { - let key = self.namespaced_key(key); - let ttl = Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - redis::cmd("TTL") - .arg(key) - .query::(connection) - .map_err(|_| Error::Unavailable) - }) - .await?; - Ok((ttl >= 0).then_some(ttl)) - } - - pub async fn async_scan_iter(&self, pattern: &str, count: usize) -> Result, Error> { - let pattern = format!("{}*", self.namespaced_key(pattern)); - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - let mut matches = Vec::new(); - connection.scan(&pattern, count, |_, keys| { - matches.extend(keys); - Ok(matches.len() < count) - })?; - matches.truncate(count); - Ok(matches) - }) - .await - } - - pub async fn async_set_cache_sadd( - &self, - key: &str, - values: Vec, - ttl: Option, - ) -> Result { - if values.is_empty() { - return Err(Error::InvalidEntry); - } - let key = self.namespaced_key(key); - let ttl = Self::ttl_seconds(ttl.unwrap_or(self.default_ttl)); - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - let mut sadd = redis::cmd("SADD"); - sadd.arg(&key).arg(values); - let mut expire = redis::cmd("EXPIRE"); - expire.arg(&key).arg(ttl); - let replies = connection.pipeline(vec![sadd, expire])?; - replies - .into_iter() - .next() - .map(redis::from_redis_value::) - .transpose() - .map_err(|_| Error::Unavailable)? - .ok_or(Error::Unavailable) - }) - .await - } - - pub async fn async_rpush(&self, key: &str, values: Vec) -> Result { - if values.is_empty() { - return Err(Error::InvalidEntry); - } - let key = self.namespaced_key(key); - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - redis::cmd("RPUSH") - .arg(key) - .arg(values) - .query(connection) - .map_err(|_| Error::Unavailable) - }) - .await - } - - pub async fn async_rpush_pipeline( - &self, - operations: Vec, - ) -> Result, Error> { - let operations = operations - .into_iter() - .map(|operation| { - if operation.values.is_empty() { - return Err(Error::InvalidEntry); - } - Ok((self.namespaced_key(&operation.key), operation.values)) - }) - .collect::, _>>()?; - if operations.is_empty() { - return Ok(Vec::new()); - } - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - let commands = operations - .into_iter() - .map(|(key, values)| { - let mut command = redis::cmd("RPUSH"); - command.arg(key).arg(values); - command - }) - .collect(); - connection - .pipeline(commands)? - .into_iter() - .map(|value| redis::from_redis_value(value).map_err(|_| Error::Unavailable)) - .collect() - }) - .await - } - - pub async fn async_lpop( - &self, - key: &str, - count: Option, - ) -> Result { - let key = self.namespaced_key(key); - let multiple = count.is_some(); - let value = Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - let mut command = redis::cmd("LPOP"); - command.arg(key); - if let Some(count) = count { - command.arg(count); - } - command - .query::(connection) - .map_err(|_| Error::Unavailable) - }) - .await?; - lpop_result(value, multiple) - } - - pub async fn async_lpop_pipeline( - &self, - operations: Vec, - ) -> Result, Error> { - let operations = operations - .into_iter() - .map(|operation| (self.namespaced_key(&operation.key), operation.count)) - .collect::>(); - if operations.is_empty() { - return Ok(Vec::new()); - } - let multiple = operations - .iter() - .map(|(_, count)| count.is_some()) - .collect::>(); - let values = Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - let commands = operations - .into_iter() - .map(|(key, count)| { - let mut command = redis::cmd("LPOP"); - command.arg(key); - if let Some(count) = count { - command.arg(count); - } - command - }) - .collect(); - connection.pipeline(commands) - }) - .await?; - values - .into_iter() - .zip(multiple) - .map(|(value, multiple)| lpop_result(value, multiple)) - .collect() - } - - pub async fn async_eval( - &self, - script: String, - keys: Vec, - arguments: Vec, - ) -> Result { - let keys = keys - .into_iter() - .map(|key| self.namespaced_key(&key)) - .collect::>(); - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - redis::cmd("EVAL") - .arg(script) - .arg(keys.len()) - .arg(keys) - .arg(arguments) - .query(connection) - .map_err(|_| Error::Unavailable) - }) - .await - } - - pub fn client_list(&self) -> Result { - self.connections - .execute(|connection| connection.node_text(redis::cmd("CLIENT").arg("LIST"))) - } - - pub fn info(&self) -> Result { - self.connections - .execute(|connection| connection.node_text(&redis::cmd("INFO"))) - } - - pub fn flushall(&self) -> Result<(), Error> { - self.connections.execute(|connection| connection.flushall()) - } -} - -impl RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - pub fn increment_with_floor( - &self, - key: &str, - amount: i64, - ttl: Duration, - ) -> Result { - let key = self.namespaced_key(key); - let ttl = Self::ttl_seconds(ttl); - self.connections - .execute(|connection| increment_with_floor(connection, key, amount, ttl)) - } - - pub async fn async_increment_pipeline( - &self, - operations: Vec, - ) -> Result, Error> { - let operations = operations - .into_iter() - .map(|operation| { - ( - self.namespaced_key(&operation.key), - operation.amount, - operation.ttl.map(Self::ttl_seconds), - ) - }) - .collect::>(); - if operations.is_empty() { - return Ok(Vec::new()); - } - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - let mut commands = Vec::with_capacity(operations.len() * 2); - let mut increments = Vec::with_capacity(operations.len()); - for (key, amount, ttl) in operations { - let mut increment = redis::cmd("INCRBYFLOAT"); - increment.arg(&key).arg(amount); - increments.push(commands.len()); - commands.push(increment); - if let Some(ttl) = ttl { - let mut expire = redis::cmd("EXPIRE"); - expire.arg(key).arg(ttl); - commands.push(expire); - } - } - let mut replies = connection.pipeline(commands)?; - increments - .into_iter() - .map(|index| { - redis::from_redis_value(std::mem::take(&mut replies[index])) - .map_err(|_| Error::Unavailable) - }) - .collect() - }) - .await - } - - pub async fn async_increment_with_floor( - &self, - key: &str, - amount: i64, - ttl: Duration, - ) -> Result { - let key = self.namespaced_key(key); - let ttl = Self::ttl_seconds(ttl); - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - increment_with_floor(connection, key, amount, ttl) - }) - .await - } - - pub async fn async_set_max( - &self, - key: &str, - value: f64, - ttl: Option, - ) -> Result { - let key = self.namespaced_key(key); - let ttl = Self::ttl_seconds(ttl.unwrap_or(self.default_ttl)); - Connections::run_blocking(Arc::clone(&self.connections), move |connection| { - redis::cmd("EVAL") - .arg(SET_MAX_SCRIPT) - .arg(1) - .arg(key) - .arg(value) - .arg(ttl) - .query(connection) - .map_err(|_| Error::Unavailable) - }) - .await - } -} - -fn redis_bytes(value: redis::Value) -> Result, Error> { - match value { - redis::Value::BulkString(bytes) => Ok(bytes), - redis::Value::SimpleString(text) => Ok(text.into_bytes()), - _ => Err(Error::InvalidEntry), - } -} - -fn lpop_result(value: redis::Value, multiple: bool) -> Result { - match value { - redis::Value::Nil => Ok(RedisLpopResult::Missing), - redis::Value::Array(values) if multiple => values - .into_iter() - .map(redis_bytes) - .collect::, _>>() - .map(RedisLpopResult::Values), - value if !multiple => redis_bytes(value).map(RedisLpopResult::Value), - _ => Err(Error::InvalidEntry), - } -} - -fn count(value: redis::Value) -> Result, Error> { - match value { - redis::Value::Nil => Ok(None), - redis::Value::Int(value) => Ok(Some(value)), - redis::Value::BulkString(value) => std::str::from_utf8(&value) - .ok() - .and_then(|value| value.parse().ok()) - .map(Some) - .ok_or(Error::InvalidEntry), - redis::Value::SimpleString(value) => { - value.parse().map(Some).map_err(|_| Error::InvalidEntry) - } - _ => Err(Error::InvalidEntry), - } -} - -fn increment_with_floor( - connection: &mut ConnectionRef<'_>, - key: String, - amount: i64, - ttl: u64, -) -> Result { - redis::cmd("EVAL") - .arg(INCREMENT_WITH_FLOOR_SCRIPT) - .arg(1) - .arg(key) - .arg(amount) - .arg(ttl) - .query(connection) - .map_err(|_| Error::Unavailable) -} - -impl TtlCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - async fn async_get_ttl(&self, key: &str) -> Result, Error> { - RedisCache::async_get_ttl(self, key) - .await - .map(|ttl| ttl.map(|seconds| Duration::from_secs(seconds as u64))) - } -} - -impl ScanCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - async fn async_scan_iter(&self, pattern: &str, count: usize) -> Result, Error> { - RedisCache::async_scan_iter(self, pattern, count).await - } -} - -impl ClientInfoCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - type ClientList = String; - type Info = String; - - fn client_list(&self) -> Result { - RedisCache::client_list(self) - } - - fn info(&self) -> Result { - RedisCache::info(self) - } -} - -impl SetCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - type SetValue = RedisArg; - type SetResult = usize; - - async fn async_set_cache_sadd( - &self, - key: &str, - values: Vec, - ttl: Option, - ) -> Result { - RedisCache::async_set_cache_sadd(self, key, values, ttl).await - } -} - -impl QueueCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - type QueueValue = RedisArg; - type PopResult = RedisLpopResult; - - async fn async_rpush(&self, key: &str, values: Vec) -> Result { - RedisCache::async_rpush(self, key, values).await - } - - async fn async_lpop(&self, key: &str, count: Option) -> Result { - RedisCache::async_lpop(self, key, count).await - } -} - -impl ScriptCache for RedisCache -where - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - type Script = RedisScript; - - fn async_register_script(&self, source: String) -> Self::Script { - RedisScript { - connections: Arc::clone(&self.connections), - namespace: self.namespace.clone(), - source, - } - } -} diff --git a/litellm-rust/crates/cache-redis/src/claim.rs b/litellm-rust/crates/cache-redis/src/claim.rs new file mode 100644 index 00000000000..e4cbe8c8d5c --- /dev/null +++ b/litellm-rust/crates/cache-redis/src/claim.rs @@ -0,0 +1,105 @@ +use litellm_cache::{CacheCodec, ClaimCache, Error, ExactCacheContext}; +use redis::Commands; + +use crate::{cache::RedisCache, connection::ConnectionRef}; + +const CLAIM_SCRIPT: &str = concat!( + "local current = redis.call('GET', KEYS[1]); ", + "if ARGV[1] == '' then if current ~= false and current ~= '' then return 0; end; ", + "elseif current ~= ARGV[1] then return 0; end; ", + "if ARGV[3] ~= '' then redis.call('SET', KEYS[1], ARGV[3], 'EX', ARGV[2]); ", + "elseif ARGV[4] == '1' then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; return 1" +); +const CLAIM_ATTEMPTS: usize = 8; + +fn stored_bytes(value: redis::Value) -> Result>, Error> { + match value { + redis::Value::Nil => Ok(None), + redis::Value::BulkString(bytes) => Ok(Some(bytes)), + redis::Value::SimpleString(text) => Ok(Some(text.into_bytes())), + _ => Err(Error::InvalidEntry), + } +} + +/// Eligibility is decided on decoded values, so a pin written by another encoder (Python's +/// `json.dumps` spacing or key order) still matches. The write is a compare-and-set on the +/// bytes that decision was made on, retried when another claimant wins the race. +fn claim( + connection: &mut ConnectionRef<'_>, + codec: &S, + key: &str, + candidate: S::Value, + eligible: &[S::Value], + ttl: u64, +) -> Result +where + S::Value: PartialEq, +{ + let payload = codec.encode(&candidate)?; + if payload.is_empty() { + return Err(Error::InvalidEntry); + } + for _ in 0..CLAIM_ATTEMPTS { + let current = stored_bytes( + connection + .get::<_, redis::Value>(key) + .map_err(|_| Error::Unavailable)?, + )? + .filter(|bytes| !bytes.is_empty()); + let existing = current + .as_deref() + .and_then(|bytes| codec.decode(bytes).ok()) + .filter(|existing| eligible.is_empty() || eligible.contains(existing)); + let refresh = existing + .as_ref() + .is_some_and(|existing| !eligible.is_empty() || *existing == candidate); + let write: &[u8] = if existing.is_some() { b"" } else { &payload }; + let applied = redis::cmd("EVAL") + .arg(CLAIM_SCRIPT) + .arg(1) + .arg(key) + .arg(current.as_deref().unwrap_or_default()) + .arg(ttl) + .arg(write) + .arg(u8::from(refresh)) + .query::(connection) + .map_err(|_| Error::Unavailable)?; + if applied { + return Ok(existing.unwrap_or(candidate)); + } + } + Err(Error::Unavailable) +} + +impl ClaimCache for RedisCache +where + S: CacheCodec + Clone + 'static, + S::Value: PartialEq, + C: redis::ConnectionLike + Send + 'static, +{ + fn claim_cache( + &self, + key: &str, + candidate: S::Value, + eligible: &[S::Value], + context: ExactCacheContext, + ) -> Result { + let key = self.namespaced_key(key); + let ttl = self.ttl_or_default(context.ttl); + self.execute(|connection| claim(connection, &self.codec, &key, candidate, eligible, ttl)) + } + + async fn async_claim_cache( + &self, + key: &str, + candidate: S::Value, + eligible: Vec, + context: ExactCacheContext, + ) -> Result { + let key = self.namespaced_key(key); + let ttl = self.ttl_or_default(context.ttl); + let codec = self.codec.clone(); + self.run(move |connection| claim(connection, &codec, &key, candidate, &eligible, ttl)) + .await + } +} diff --git a/litellm-rust/crates/cache-redis/src/cache/connection.rs b/litellm-rust/crates/cache-redis/src/connection.rs similarity index 65% rename from litellm-rust/crates/cache-redis/src/cache/connection.rs rename to litellm-rust/crates/cache-redis/src/connection.rs index 013bf055f89..2f58e2a9b80 100644 --- a/litellm-rust/crates/cache-redis/src/cache/connection.rs +++ b/litellm-rust/crates/cache-redis/src/connection.rs @@ -1,29 +1,126 @@ -use std::collections::HashMap; +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; use litellm_cache::Error; use redis::{ - ConnectionAddr, ConnectionInfo, ConnectionLike, IntoConnectionInfo, - cluster::{ClusterClient, ClusterClientBuilder, ClusterConnection, NodeAddress}, + ConnectionAddr, ConnectionInfo, IntoConnectionInfo, + cluster::{ + ClusterClient, ClusterClientBuilder, ClusterConnection, ClusterPipeline, NodeAddress, + }, cluster_routing::{ - MultipleNodeRoutingInfo, ResponsePolicy, RoutingInfo, SingleNodeRoutingInfo, Slot, + MultipleNodeRoutingInfo, ResponsePolicy, RoutingInfo, SingleNodeRoutingInfo, }, }; -use super::REDIS_TIMEOUT; -use crate::topology::RedisNode; +use crate::topology::{RedisNode, RedisTopology}; -pub struct PooledConnection { - pub(super) connection: C, - pub(super) failed: bool, +pub(crate) const REDIS_TIMEOUT: Duration = Duration::from_secs(5); +const REDIS_POOL_SIZE: u32 = 16; + +#[allow(private_interfaces)] +pub enum Connections { + Pool(r2d2::Pool), + Cluster(r2d2::Pool), + Fixed(Mutex), +} + +impl Connections +where + C: redis::ConnectionLike + Send + 'static, +{ + pub fn execute( + &self, + operation: impl FnOnce(&mut ConnectionRef<'_>) -> Result, + ) -> Result { + match self { + Self::Pool(pool) => { + let mut pooled = pool.get().map_err(|_| Error::Unavailable)?; + let result = operation(&mut ConnectionRef::Node(&mut pooled.connection)); + pooled.failed = matches!(result, Err(Error::Unavailable)); + result + } + Self::Cluster(pool) => { + let mut pooled = pool.get().map_err(|_| Error::Unavailable)?; + let result = operation(&mut ConnectionRef::Cluster(&mut pooled.connection)); + pooled.failed = matches!(result, Err(Error::Unavailable)); + result + } + Self::Fixed(connection) => { + let mut connection = connection.lock().map_err(|_| Error::Unavailable)?; + operation(&mut ConnectionRef::Node(&mut *connection)) + } + } + } + + pub async fn run_blocking(connections: Arc, operation: F) -> Result + where + T: Send + 'static, + F: FnOnce(&mut ConnectionRef<'_>) -> Result + Send + 'static, + { + tokio::task::spawn_blocking(move || connections.execute(operation)) + .await + .map_err(|_| Error::Unavailable)? + } + + pub fn fixed(connection: C) -> Self { + Self::Fixed(Mutex::new(connection)) + } + + pub fn open(url: &str, topology: &RedisTopology) -> Result { + match topology { + RedisTopology::Standalone => Ok(Self::Pool(pool(ConnectionManager::open(url)?)?)), + RedisTopology::Cluster { startup_nodes } => Ok(Self::Cluster(pool( + ClusterConnectionManager::open(url, startup_nodes)?, + )?)), + } + } + + /// Closes every idle pooled connection; the next operation opens a fresh one. Connections + /// checked out right now return to the pool, and a caller-owned connection stays open. + pub fn disconnect(&self) { + match self { + Self::Pool(pool) => close_idle(pool), + Self::Cluster(pool) => close_idle(pool), + Self::Fixed(_) => {} + } + } +} + +fn pool(manager: M) -> Result, Error> { + r2d2::Pool::builder() + .max_size(REDIS_POOL_SIZE) + .min_idle(Some(0)) + .connection_timeout(REDIS_TIMEOUT) + .test_on_check_out(false) + .build(manager) + .map_err(|_| Error::Unavailable) +} + +fn close_idle(pool: &r2d2::Pool) +where + M: r2d2::ManageConnection>, +{ + let mut idle = Vec::new(); + while let Some(mut connection) = pool.try_get() { + connection.failed = true; + idle.push(connection); + } +} + +pub(crate) struct PooledConnection { + connection: C, + failed: bool, } /// Pools connections without a checkout PING, which would double every operation's round trips. /// A timed-out command leaves its reply on the socket while redis still reports the connection /// open, so any connection whose operation failed is discarded instead of being reused. -pub struct ConnectionManager(redis::Client); +pub(crate) struct ConnectionManager(redis::Client); impl ConnectionManager { - pub(super) fn open(url: &str) -> Result { + fn open(url: &str) -> Result { redis::Client::open(url) .map(Self) .map_err(|_| Error::Unavailable) @@ -54,10 +151,10 @@ impl r2d2::ManageConnection for ConnectionManager { } } -pub struct ClusterConnectionManager(ClusterClient); +pub(crate) struct ClusterConnectionManager(ClusterClient); impl ClusterConnectionManager { - pub(super) fn open(url: &str, startup_nodes: &[RedisNode]) -> Result { + fn open(url: &str, startup_nodes: &[RedisNode]) -> Result { if startup_nodes.is_empty() { return Err(Error::Unavailable); } @@ -172,43 +269,36 @@ impl redis::ConnectionLike for ConnectionRef<'_> { } impl ConnectionRef<'_> { - pub(crate) fn pipeline( + /// Runs `pipeline` and decodes its non-ignored replies as `T`. A cluster connection refuses + /// `Pipeline::query`, so there a transaction goes to its keys' slot as one MULTI/EXEC and + /// anything else is split per node by `ClusterPipeline`; either way the raw replies are + /// handed back to `pipeline` to decode. + pub(crate) fn query_pipeline( &mut self, - commands: Vec, - ) -> Result, Error> { + pipeline: &redis::Pipeline, + ) -> Result { match self { - Self::Node(connection) => { - let mut pipeline = redis::pipe(); - for command in &commands { - pipeline.add_command(command.clone()); - } - pipeline - .query::>(*connection) - .map_err(|_| Error::Unavailable) + Self::Node(connection) => pipeline.query(*connection), + Self::Cluster(connection) if pipeline.is_transaction() => { + redis::ConnectionLike::req_packed_commands( + *connection, + &pipeline.get_packed_pipeline(), + pipeline.len() + 1, + 1, + ) + .and_then(|replies| pipeline.query(&mut Replies(Some(replies)))) } Self::Cluster(connection) => { - let mut replies: Vec> = vec![None; commands.len()]; - for indices in slot_groups(&commands).into_values() { - let mut pipeline = redis::pipe(); - for index in &indices { - pipeline.add_command(commands[*index].clone()); - } - let values = connection - .req_packed_commands(&pipeline.get_packed_pipeline(), 0, indices.len()) - .map_err(|_| Error::Unavailable)?; - if values.len() != indices.len() { - return Err(Error::Unavailable); - } - for (index, value) in indices.into_iter().zip(values) { - replies[index] = Some(value); - } + let mut cluster = ClusterPipeline::with_capacity(pipeline.len()); + for command in pipeline.cmd_iter() { + cluster.add_command(command.clone()); } - replies - .into_iter() - .collect::>>() - .ok_or(Error::Unavailable) + cluster + .query(connection) + .and_then(|replies| pipeline.query(&mut Replies(Some(replies)))) } } + .map_err(|_| Error::Unavailable) } pub(crate) fn scan( @@ -379,14 +469,35 @@ fn scan_command(cursor: u64, pattern: &str, count: usize) -> redis::Cmd { command } -fn slot_groups(commands: &[redis::Cmd]) -> HashMap> { - let mut groups: HashMap> = HashMap::new(); - for (index, command) in commands.iter().enumerate() { - let key = match command.args_iter().nth(1) { - Some(redis::Arg::Simple(key)) => key, - _ => b"", - }; - groups.entry(Slot::for_key(key)).or_default().push(index); +/// Hands already received pipeline replies to `Pipeline::query`, so it applies its own +/// ignore and error handling to replies a cluster pipeline gathered from several nodes. +struct Replies(Option>); + +impl redis::ConnectionLike for Replies { + fn req_packed_command(&mut self, _: &[u8]) -> redis::RedisResult { + Err((redis::ErrorKind::Client, "replies hold a pipeline only").into()) + } + + fn req_packed_commands( + &mut self, + _: &[u8], + _: usize, + _: usize, + ) -> redis::RedisResult> { + self.0 + .take() + .ok_or_else(|| (redis::ErrorKind::Client, "replies were already read").into()) + } + + fn get_db(&self) -> i64 { + 0 + } + + fn check_connection(&mut self) -> bool { + true + } + + fn is_open(&self) -> bool { + true } - groups } diff --git a/litellm-rust/crates/cache-redis/src/counter.rs b/litellm-rust/crates/cache-redis/src/counter.rs new file mode 100644 index 00000000000..adcd597b2f4 --- /dev/null +++ b/litellm-rust/crates/cache-redis/src/counter.rs @@ -0,0 +1,205 @@ +use std::time::Duration; + +use litellm_cache::{ + BoundedCounterCache, CacheCodec, CountReadCache, CounterCache, Error, ExactCacheContext, + IncrementOperation, +}; + +use crate::{ + cache::{RedisCache, ttl_seconds}, + connection::ConnectionRef, + store::mget, +}; + +const INCREMENT_SCRIPT: &str = concat!( + "local value = redis.call('INCRBYFLOAT', KEYS[1], ARGV[1]); ", + "if redis.call('TTL', KEYS[1]) == -1 then ", + "redis.call('EXPIRE', KEYS[1], ARGV[2]); end; return value" +); +const INCREMENT_WITH_FLOOR_SCRIPT: &str = concat!( + "local count = redis.call('INCRBY', KEYS[1], ARGV[1]); ", + "if count < 0 then count = redis.call('INCRBY', KEYS[1], -count); end; ", + "if redis.call('TTL', KEYS[1]) < 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; ", + "return count" +); +const SET_MAX_SCRIPT: &str = concat!( + "local current = redis.call('GET', KEYS[1]); ", + "if current == false or tonumber(current) < tonumber(ARGV[1]) then ", + "redis.call('SET', KEYS[1], ARGV[1]); ", + "if tonumber(ARGV[2]) > 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; ", + "return ARGV[1]; end; return current" +); + +impl CounterCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + fn increment_cache( + &self, + key: &str, + amount: f64, + context: ExactCacheContext, + ) -> Result { + let key = self.namespaced_key(key); + let ttl = self.ttl_or_default(context.ttl); + self.execute(|connection| increment(connection, key, amount, ttl, false)) + } + + /// Python `_incrbyfloat_with_ttl`: without `refresh_ttl` the TTL is set only on a key that + /// has none, in one atomic script; with it, every increment re-arms the TTL. + async fn async_increment( + &self, + key: &str, + amount: f64, + context: ExactCacheContext, + refresh_ttl: bool, + ) -> Result { + let key = self.namespaced_key(key); + let ttl = self.ttl_or_default(context.ttl); + self.run(move |connection| increment(connection, key, amount, ttl, refresh_ttl)) + .await + } + + async fn async_increment_pipeline( + &self, + operations: Vec, + ) -> Result, Error> { + if operations.is_empty() { + return Ok(Vec::new()); + } + let mut pipeline = redis::pipe(); + for operation in operations { + let key = self.namespaced_key(&operation.key); + pipeline.cmd("INCRBYFLOAT").arg(&key).arg(operation.amount); + if let Some(ttl) = operation.ttl { + pipeline + .cmd("EXPIRE") + .arg(key) + .arg(ttl_seconds(ttl)) + .ignore(); + } + } + self.run(move |connection| connection.query_pipeline(&pipeline)) + .await + } +} + +fn increment( + connection: &mut ConnectionRef<'_>, + key: String, + amount: f64, + ttl: u64, + refresh_ttl: bool, +) -> Result { + if !refresh_ttl { + return redis::cmd("EVAL") + .arg(INCREMENT_SCRIPT) + .arg(1) + .arg(key) + .arg(amount) + .arg(ttl) + .query(connection) + .map_err(|_| Error::Unavailable); + } + connection + .query_pipeline( + redis::pipe() + .cmd("INCRBYFLOAT") + .arg(&key) + .arg(amount) + .cmd("EXPIRE") + .arg(&key) + .arg(ttl) + .ignore(), + ) + .map(|(value,)| value) +} + +impl CountReadCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + fn batch_get_counts(&self, keys: &[String]) -> Result>, Error> { + let keys = self.namespaced_keys(keys); + self.execute(|connection| mget(connection, keys))? + .into_iter() + .map(count) + .collect() + } + + async fn async_batch_get_counts(&self, keys: Vec) -> Result>, Error> { + let keys = self.namespaced_keys(&keys); + self.run(move |connection| mget(connection, keys)) + .await? + .into_iter() + .map(count) + .collect() + } +} + +fn count(value: redis::Value) -> Result, Error> { + redis::from_redis_value(value).map_err(|_| Error::InvalidEntry) +} + +impl BoundedCounterCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + fn increment_with_floor(&self, key: &str, amount: i64, ttl: Duration) -> Result { + let key = self.namespaced_key(key); + let ttl = ttl_seconds(ttl); + self.execute(|connection| increment_with_floor(connection, key, amount, ttl)) + } + + async fn async_increment_with_floor( + &self, + key: &str, + amount: i64, + ttl: Duration, + ) -> Result { + let key = self.namespaced_key(key); + let ttl = ttl_seconds(ttl); + self.run(move |connection| increment_with_floor(connection, key, amount, ttl)) + .await + } + + async fn async_set_max( + &self, + key: &str, + value: f64, + ttl: Option, + ) -> Result { + let key = self.namespaced_key(key); + let ttl = self.ttl_or_default(ttl); + self.run(move |connection| { + redis::cmd("EVAL") + .arg(SET_MAX_SCRIPT) + .arg(1) + .arg(key) + .arg(value) + .arg(ttl) + .query(connection) + .map_err(|_| Error::Unavailable) + }) + .await + } +} + +fn increment_with_floor( + connection: &mut ConnectionRef<'_>, + key: String, + amount: i64, + ttl: u64, +) -> Result { + redis::cmd("EVAL") + .arg(INCREMENT_WITH_FLOOR_SCRIPT) + .arg(1) + .arg(key) + .arg(amount) + .arg(ttl) + .query(connection) + .map_err(|_| Error::Unavailable) +} diff --git a/litellm-rust/crates/cache-redis/src/keys.rs b/litellm-rust/crates/cache-redis/src/keys.rs new file mode 100644 index 00000000000..b9d7bb20ff3 --- /dev/null +++ b/litellm-rust/crates/cache-redis/src/keys.rs @@ -0,0 +1,63 @@ +use std::time::Duration; + +use litellm_cache::{CacheCodec, Error, RefreshTtlCache, ScanCache, TtlCache}; + +use crate::cache::RedisCache; + +impl TtlCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + async fn async_get_ttl(&self, key: &str) -> Result, Error> { + let key = self.namespaced_key(key); + let ttl = self + .run(move |connection| { + redis::cmd("TTL") + .arg(key) + .query::(connection) + .map_err(|_| Error::Unavailable) + }) + .await?; + Ok(u64::try_from(ttl).ok().map(Duration::from_secs)) + } +} + +impl RefreshTtlCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + async fn async_refresh_ttl(&self, key: &str, ttl: Option) -> Result { + let key = self.namespaced_key(key); + let ttl = self.ttl_or_default(ttl); + self.run(move |connection| { + redis::cmd("EXPIRE") + .arg(key) + .arg(ttl) + .query(connection) + .map_err(|_| Error::Unavailable) + }) + .await + } +} + +impl ScanCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + async fn async_scan_iter(&self, pattern: &str, count: usize) -> Result, Error> { + let pattern = format!("{}*", self.namespaced_key(pattern)); + self.run(move |connection| { + let mut matches = Vec::new(); + connection.scan(&pattern, count, |_, keys| { + matches.extend(keys); + Ok(matches.len() < count) + })?; + matches.truncate(count); + Ok(matches) + }) + .await + } +} diff --git a/litellm-rust/crates/cache-redis/src/lib.rs b/litellm-rust/crates/cache-redis/src/lib.rs index efb0db931ac..037e39e5d40 100644 --- a/litellm-rust/crates/cache-redis/src/lib.rs +++ b/litellm-rust/crates/cache-redis/src/lib.rs @@ -1,11 +1,15 @@ mod cache; +mod claim; +pub mod connection; +mod counter; +mod keys; +mod lifecycle; +mod queue; +mod script; +mod store; mod topology; -pub mod connection { - pub use crate::cache::{ConnectionRef, Connections}; -} - -pub use cache::{ - RedisArg, RedisCache, RedisLpopOperation, RedisLpopResult, RedisRpushOperation, RedisScript, -}; +pub use cache::RedisCache; +pub use queue::{RedisLpopOperation, RedisLpopResult, RedisRpushOperation}; +pub use script::{RedisArg, RedisScript}; pub use topology::{RedisNode, RedisTopology}; diff --git a/litellm-rust/crates/cache-redis/src/lifecycle.rs b/litellm-rust/crates/cache-redis/src/lifecycle.rs new file mode 100644 index 00000000000..6ab8c31509e --- /dev/null +++ b/litellm-rust/crates/cache-redis/src/lifecycle.rs @@ -0,0 +1,85 @@ +use litellm_cache::{ + CacheCodec, CacheConnectionResult, CacheConnectionStatus, ClientInfoCache, ConnectionCache, + DisconnectCache, Error, PingCache, +}; + +use crate::{cache::RedisCache, topology::RedisTopology}; + +impl PingCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + fn sync_ping(&self) -> Result { + self.execute(|connection| connection.ping().map_err(|_| Error::Unavailable)) + } + + async fn ping(&self) -> Result { + self.run(|connection| connection.ping().map_err(|_| Error::Unavailable)) + .await + } +} + +impl ConnectionCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + /// Python `RedisCache.test_connection`, or `RedisClusterCache.test_connection` for a + /// cluster topology, which differs only in its messages. + async fn test_connection(&self) -> Result { + let label = match self.topology { + RedisTopology::Standalone => "Redis", + RedisTopology::Cluster { .. } => "Redis Cluster", + }; + let ping = self + .run(|connection| Ok(connection.ping().map_err(|error| error.to_string()))) + .await + .unwrap_or_else(|error| Err(error.to_string())); + Ok(match ping { + Ok(true) => CacheConnectionResult { + status: CacheConnectionStatus::Success, + message: format!("{label} connection test successful"), + error: None, + }, + Ok(false) => CacheConnectionResult { + status: CacheConnectionStatus::Failed, + message: format!("{label} ping returned False"), + error: None, + }, + Err(error) => CacheConnectionResult { + status: CacheConnectionStatus::Failed, + message: format!("{label} connection failed: {error}"), + error: Some(error), + }, + }) + } +} + +impl DisconnectCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + async fn disconnect(&self) -> Result<(), Error> { + self.connections.disconnect(); + Ok(()) + } +} + +impl ClientInfoCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + type ClientList = String; + type Info = String; + + fn client_list(&self) -> Result { + self.execute(|connection| connection.node_text(redis::cmd("CLIENT").arg("LIST"))) + } + + fn info(&self) -> Result { + self.execute(|connection| connection.node_text(&redis::cmd("INFO"))) + } +} diff --git a/litellm-rust/crates/cache-redis/src/queue.rs b/litellm-rust/crates/cache-redis/src/queue.rs new file mode 100644 index 00000000000..623bce23640 --- /dev/null +++ b/litellm-rust/crates/cache-redis/src/queue.rs @@ -0,0 +1,226 @@ +use std::time::Duration; + +use litellm_cache::{CacheCodec, Error, PopOperation, PushOperation, QueueCache, SetCache}; + +use crate::{cache::RedisCache, script::RedisArg}; + +pub type RedisRpushOperation = PushOperation; +pub type RedisLpopOperation = PopOperation; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RedisLpopResult { + Missing, + Value(Vec), + Values(Vec>), +} + +impl SetCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + type SetValue = RedisArg; + type SetResult = usize; + + async fn async_set_cache_sadd( + &self, + key: &str, + values: Vec, + ttl: Option, + ) -> Result { + if values.is_empty() { + return Err(Error::InvalidEntry); + } + let key = self.namespaced_key(key); + let ttl = self.ttl_or_default(ttl); + let mut pipeline = redis::pipe(); + pipeline + .cmd("SADD") + .arg(&key) + .arg(values) + .cmd("EXPIRE") + .arg(&key) + .arg(ttl) + .ignore(); + self.run(move |connection| connection.query_pipeline(&pipeline)) + .await + .map(|(added,)| added) + } +} + +impl QueueCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + type QueueValue = RedisArg; + type PopResult = RedisLpopResult; + + async fn async_rpush(&self, key: &str, values: Vec) -> Result { + if values.is_empty() { + return Err(Error::InvalidEntry); + } + let key = self.namespaced_key(key); + self.run(move |connection| { + redis::cmd("RPUSH") + .arg(key) + .arg(values) + .query(connection) + .map_err(|_| Error::Unavailable) + }) + .await + } + + async fn async_rpush_and_trim( + &self, + key: &str, + values: Vec, + max_len: usize, + ) -> Result { + if values.is_empty() { + return Err(Error::InvalidEntry); + } + let key = self.namespaced_key(key); + let start = i64::try_from(max_len).map_or(i64::MIN, |max_len| -max_len); + let mut pipeline = redis::pipe(); + pipeline + .atomic() + .cmd("RPUSH") + .arg(&key) + .arg(values) + .cmd("LTRIM") + .arg(&key) + .arg(start) + .arg(-1) + .ignore(); + self.run(move |connection| connection.query_pipeline(&pipeline)) + .await + .map(|(length,)| length) + } + + async fn async_rpush_pipeline( + &self, + operations: Vec, + ) -> Result, Error> { + if operations.is_empty() { + return Ok(Vec::new()); + } + let mut pipeline = redis::pipe(); + for operation in operations { + if operation.values.is_empty() { + return Err(Error::InvalidEntry); + } + pipeline + .cmd("RPUSH") + .arg(self.namespaced_key(&operation.key)) + .arg(operation.values); + } + self.run(move |connection| connection.query_pipeline(&pipeline)) + .await + } + + async fn async_lpop(&self, key: &str, count: Option) -> Result { + if let Some(count) = count + && self.major_version().await < 7 + { + return self.lpop_one_at_a_time(key, count).await; + } + let command = lpop(self.namespaced_key(key), count); + let value = self + .run(move |connection| { + command + .query::(connection) + .map_err(|_| Error::Unavailable) + }) + .await?; + lpop_result(value, count.is_some()) + } + + async fn async_lpop_pipeline( + &self, + operations: Vec, + ) -> Result, Error> { + if operations.is_empty() { + return Ok(Vec::new()); + } + if operations.iter().any(|operation| operation.count.is_some()) + && self.major_version().await < 7 + { + let mut results = Vec::with_capacity(operations.len()); + for operation in &operations { + results.push(self.async_lpop(&operation.key, operation.count).await?); + } + return Ok(results); + } + let multiple = operations + .iter() + .map(|operation| operation.count.is_some()) + .collect::>(); + let mut pipeline = redis::pipe(); + for operation in operations { + pipeline.add_command(lpop(self.namespaced_key(&operation.key), operation.count)); + } + self.run(move |connection| connection.query_pipeline::>(&pipeline)) + .await? + .into_iter() + .zip(multiple) + .map(|(value, multiple)| lpop_result(value, multiple)) + .collect() + } +} + +fn lpop(key: String, count: Option) -> redis::Cmd { + let mut command = redis::cmd("LPOP"); + command.arg(key); + if let Some(count) = count { + command.arg(count); + } + command +} + +fn redis_bytes(value: redis::Value) -> Result, Error> { + match value { + redis::Value::BulkString(bytes) => Ok(bytes), + redis::Value::SimpleString(text) => Ok(text.into_bytes()), + _ => Err(Error::InvalidEntry), + } +} + +impl RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + /// `handle_lpop_count_for_older_redis_versions`: `count` single-`LPOP` pipelines, keeping + /// only the values actually popped. + async fn lpop_one_at_a_time(&self, key: &str, count: usize) -> Result { + let key = self.namespaced_key(key); + let mut values = Vec::new(); + for _ in 0..count { + let mut pipeline = redis::pipe(); + pipeline.add_command(lpop(key.clone(), None)); + let replies = self + .run(move |connection| connection.query_pipeline::>(&pipeline)) + .await?; + for reply in replies { + if reply != redis::Value::Nil { + values.push(redis_bytes(reply)?); + } + } + } + Ok(RedisLpopResult::Values(values)) + } +} + +fn lpop_result(value: redis::Value, multiple: bool) -> Result { + match value { + redis::Value::Nil => Ok(RedisLpopResult::Missing), + redis::Value::Array(values) if multiple => values + .into_iter() + .map(redis_bytes) + .collect::, _>>() + .map(RedisLpopResult::Values), + value if !multiple => redis_bytes(value).map(RedisLpopResult::Value), + _ => Err(Error::InvalidEntry), + } +} diff --git a/litellm-rust/crates/cache-redis/src/script.rs b/litellm-rust/crates/cache-redis/src/script.rs new file mode 100644 index 00000000000..e8a16bcc2b5 --- /dev/null +++ b/litellm-rust/crates/cache-redis/src/script.rs @@ -0,0 +1,136 @@ +use std::sync::Arc; + +use litellm_cache::{CacheCodec, CacheScript, Error, ScriptCache}; + +use crate::{ + cache::{RedisCache, namespaced_key}, + connection::Connections, +}; + +#[derive(Clone, Debug, PartialEq)] +pub enum RedisArg { + Bytes(Vec), + Integer(i64), + Float(f64), +} + +impl From<&str> for RedisArg { + fn from(value: &str) -> Self { + Self::Bytes(value.as_bytes().to_vec()) + } +} + +impl From for RedisArg { + fn from(value: String) -> Self { + Self::Bytes(value.into_bytes()) + } +} + +impl From> for RedisArg { + fn from(value: Vec) -> Self { + Self::Bytes(value) + } +} + +impl From for RedisArg { + fn from(value: i64) -> Self { + Self::Integer(value) + } +} + +impl From for RedisArg { + fn from(value: f64) -> Self { + Self::Float(value) + } +} + +impl redis::ToRedisArgs for RedisArg { + fn write_redis_args(&self, out: &mut W) + where + W: ?Sized + redis::RedisWrite, + { + match self { + Self::Bytes(value) => value.write_redis_args(out), + Self::Integer(value) => value.write_redis_args(out), + Self::Float(value) => value.write_redis_args(out), + } + } +} + +pub struct RedisScript { + connections: Arc>, + namespace: Option, + source: String, +} + +impl CacheScript for RedisScript +where + C: redis::ConnectionLike + Send + 'static, +{ + type Argument = RedisArg; + type Output = redis::Value; + + async fn invoke( + &self, + keys: Vec, + arguments: Vec, + ) -> Result { + let keys = keys + .into_iter() + .map(|key| namespaced_key(self.namespace.as_deref(), &key)) + .collect::>(); + let source = self.source.clone(); + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { + eval(connection, &source, keys, arguments) + }) + .await + } +} + +impl RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + pub async fn async_eval( + &self, + script: String, + keys: Vec, + arguments: Vec, + ) -> Result { + let keys = self.namespaced_keys(&keys); + self.run(move |connection| eval(connection, &script, keys, arguments)) + .await + } +} + +fn eval( + connection: &mut impl redis::ConnectionLike, + script: &str, + keys: Vec, + arguments: Vec, +) -> Result { + redis::cmd("EVAL") + .arg(script) + .arg(keys.len()) + .arg(keys) + .arg(arguments) + .query(connection) + .map_err(|_| Error::Unavailable) +} + +impl ScriptCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + type Script = RedisScript; + + fn async_register_script(&self, source: String) -> Self::Script { + RedisScript { + connections: Arc::clone(&self.connections), + namespace: self.namespace.clone(), + source, + } + } +} diff --git a/litellm-rust/crates/cache-redis/src/store.rs b/litellm-rust/crates/cache-redis/src/store.rs new file mode 100644 index 00000000000..111d481ad02 --- /dev/null +++ b/litellm-rust/crates/cache-redis/src/store.rs @@ -0,0 +1,232 @@ +use std::time::Duration; + +use litellm_cache::{ + BaseCache, BatchCache, BatchEntry, BulkDeleteCache, CacheCodec, DeleteCache, Error, + ExactCacheContext, FlushAllCache, FlushCache, TtlPipelineCache, +}; +use redis::Commands; + +use crate::{cache::RedisCache, connection::ConnectionRef}; + +impl BaseCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + type Value = S::Value; + type Context = ExactCacheContext; + + fn get_ttl(&self, context: &Self::Context) -> Option { + context.ttl.or(Some(self.default_ttl)) + } + + fn set_cache( + &self, + key: &str, + value: Self::Value, + context: &ExactCacheContext, + ) -> Result<(), Error> { + let payload = self.codec.encode(&value)?; + let ttl = self.ttl_or_default(context.ttl); + let key = self.namespaced_key(key); + self.execute(|connection| { + connection + .set_ex::<_, _, ()>(key, payload, ttl) + .map_err(|_| Error::Unavailable) + }) + } + + fn get_cache(&self, key: &str, _: &ExactCacheContext) -> Result, Error> { + let key = self.namespaced_key(key); + let value = self.execute(|connection| { + connection + .get::<_, redis::Value>(key) + .map_err(|_| Error::Unavailable) + })?; + self.decode_response(value) + } + + async fn async_set_cache( + &self, + key: &str, + value: Self::Value, + context: ExactCacheContext, + ) -> Result<(), Error> { + let payload = self.codec.encode(&value)?; + let key = self.namespaced_key(key); + let ttl = self.ttl_or_default(context.ttl); + self.run(move |connection| { + connection + .set_ex::<_, _, ()>(key, payload, ttl) + .map_err(|_| Error::Unavailable) + }) + .await + } + + async fn async_get_cache( + &self, + key: &str, + _: &ExactCacheContext, + ) -> Result, Error> { + let key = self.namespaced_key(key); + let value = self + .run(move |connection| { + connection + .get::<_, redis::Value>(key) + .map_err(|_| Error::Unavailable) + }) + .await?; + self.decode_response(value) + } + + async fn async_set_cache_pipeline( + &self, + cache_list: Vec<(String, Self::Value)>, + context: ExactCacheContext, + ) -> Result<(), Error> { + self.async_set_cache_pipeline_with_ttls( + cache_list + .into_iter() + .map(|(key, value)| (key, value, context.ttl)) + .collect(), + ) + .await + } +} + +impl TtlPipelineCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + async fn async_set_cache_pipeline_with_ttls( + &self, + entries: Vec<(String, Self::Value, Option)>, + ) -> Result<(), Error> { + if entries.is_empty() { + return Ok(()); + } + let mut pipeline = redis::pipe(); + for (key, value, ttl) in entries { + pipeline + .cmd("SETEX") + .arg(self.namespaced_key(&key)) + .arg(self.ttl_or_default(ttl)) + .arg(self.codec.encode(&value)?) + .ignore(); + } + self.run(move |connection| connection.query_pipeline(&pipeline)) + .await + } +} + +impl BatchCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + fn batch_get_cache( + &self, + keys: &[String], + _: &ExactCacheContext, + ) -> Result>, Error> { + let keys = self.namespaced_keys(keys); + self.execute(|connection| mget(connection, keys))? + .into_iter() + .map(|value| self.decode_batch_response(value)) + .collect() + } + + async fn async_batch_get_cache( + &self, + keys: Vec, + _: ExactCacheContext, + ) -> Result>, Error> { + let keys = self.namespaced_keys(&keys); + self.run(move |connection| mget(connection, keys)) + .await? + .into_iter() + .map(|value| self.decode_batch_response(value)) + .collect() + } +} + +pub(crate) fn mget( + connection: &mut ConnectionRef<'_>, + keys: Vec, +) -> Result, Error> { + redis::cmd("MGET") + .arg(keys) + .query(connection) + .map_err(|_| Error::Unavailable) +} + +impl DeleteCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + fn delete_cache(&self, key: &str) -> Result<(), Error> { + let key = self.namespaced_key(key); + self.execute(|connection| connection.del::<_, ()>(key).map_err(|_| Error::Unavailable)) + } + + async fn async_delete_cache(&self, key: &str) -> Result<(), Error> { + let key = self.namespaced_key(key); + self.run(move |connection| connection.del::<_, ()>(key).map_err(|_| Error::Unavailable)) + .await + } +} + +impl BulkDeleteCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + async fn delete_cache_keys(&self, keys: Vec) -> Result { + if keys.is_empty() { + return Ok(0); + } + let keys = self.namespaced_keys(&keys); + self.run(move |connection| connection.del(keys).map_err(|_| Error::Unavailable)) + .await + } +} + +impl FlushCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + fn flush_cache(&self) -> Result<(), Error> { + let pattern = self.namespaced_pattern()?; + self.execute(|connection| flush_matching(connection, &pattern)) + } + + async fn async_flush_cache(&self) -> Result<(), Error> { + let pattern = self.namespaced_pattern()?; + self.run(move |connection| flush_matching(connection, &pattern)) + .await + } +} + +impl FlushAllCache for RedisCache +where + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + fn flushall(&self) -> Result<(), Error> { + self.execute(|connection| connection.flushall()) + } +} + +fn flush_matching(connection: &mut ConnectionRef<'_>, pattern: &str) -> Result<(), Error> { + connection.scan(pattern, 1000, |connection, keys| { + if !keys.is_empty() { + connection + .del::<_, usize>(keys) + .map_err(|_| Error::Unavailable)?; + } + Ok(true) + }) +} diff --git a/litellm-rust/crates/cache-redis/tests/cache.rs b/litellm-rust/crates/cache-redis/tests/cache.rs index 337f27984f8..70baeffd572 100644 --- a/litellm-rust/crates/cache-redis/tests/cache.rs +++ b/litellm-rust/crates/cache-redis/tests/cache.rs @@ -1,43 +1,84 @@ +mod support; + use std::time::Duration; use litellm_cache::{ - BaseCache, BatchCache, BatchEntry, CacheCodec, CacheConnectionStatus, CacheScript, ClaimCache, - CounterCache, DeleteCache, Error, ExactCacheContext, FlushCache, IncrementOperation, JsonCodec, - ScriptCache, get_cache, set_cache, + BaseCache, BatchCache, BatchEntry, BoundedCounterCache, BulkDeleteCache, CacheCodec, + CacheConnectionStatus, CacheScript, ClaimCache, ClientInfoCache, ConnectionCache, + CountReadCache, CounterCache, DeleteCache, DisconnectCache, Error, ExactCacheContext, + FlushAllCache, FlushCache, IncrementOperation, JsonCodec, PingCache, QueueCache, + RefreshTtlCache, ScanCache, ScriptCache, SetCache, TtlCache, TtlPipelineCache, get_cache, + set_cache, }; use litellm_cache_redis::{ RedisArg, RedisCache, RedisLpopOperation, RedisLpopResult, RedisRpushOperation, }; use redis_test::{MockCmd, MockRedisConnection}; +use rstest::{fixture, rstest}; +use serde_json::json; +use support::TaggedByteCodec; -struct TaggedByteCodec(u8); +type Mocked = RedisCache; -impl CacheCodec for TaggedByteCodec { - type Value = u8; - - fn encode(&self, value: &u8) -> Result, Error> { - if *value > 127 { - return Err(Error::InvalidEntry); - } - Ok(vec![self.0, *value]) - } - - fn decode(&self, bytes: &[u8]) -> Result { - match bytes { - [tag, value] if *tag == self.0 => Ok(*value), - _ => Err(Error::InvalidEntry), - } - } +fn mock(commands: Vec) -> MockRedisConnection { + MockRedisConnection::new(commands).assert_all_commands_consumed() } -#[test] +fn tagged(commands: Vec) -> Mocked { + RedisCache::with_connection(mock(commands), None, TaggedByteCodec(42)) +} + +fn json_cache(commands: Vec) -> Mocked> { + RedisCache::with_connection(mock(commands), None, JsonCodec::new()) +} + +fn team(commands: Vec) -> Mocked> { + json_cache(commands).with_namespace(Some("team".into())) +} + +#[fixture] +fn context() -> ExactCacheContext { + ExactCacheContext::default() +} + +fn scan(pattern: &str, cursor: u64, count: usize, reply: redis::Value) -> MockCmd { + MockCmd::new( + redis::cmd("SCAN") + .cursor_arg(cursor) + .arg("MATCH") + .arg(pattern) + .arg("COUNT") + .arg(count), + Ok(reply), + ) +} + +#[rstest] fn constructor_rejects_invalid_urls() { assert!(RedisCache::new("not a redis url", None, JsonCodec::::new()).is_err()); } -#[test] +#[rstest] +#[case::zero_rounds_up_to_one(Some(Duration::ZERO), 1)] +#[case::fractions_round_up(Some(Duration::from_millis(1500)), 2)] +#[case::whole_seconds_are_kept(Some(Duration::from_secs(15)), 15)] +#[case::missing_ttl_uses_default(None, 600)] +fn writes_round_ttls_up_to_positive_seconds(#[case] ttl: Option, #[case] seconds: u64) { + let cache = tagged(vec![MockCmd::new( + redis::cmd("SETEX") + .arg("key") + .arg(seconds) + .arg([42u8, 7].as_slice()), + Ok("OK"), + )]); + cache + .set_cache("key", 7, &ExactCacheContext { ttl }) + .unwrap(); +} + +#[rstest] fn generic_helpers_use_the_injected_codec_and_ttl() { - let connection = MockRedisConnection::new([ + let cache = tagged(vec![ MockCmd::new( redis::cmd("SETEX") .arg("counter") @@ -46,9 +87,7 @@ fn generic_helpers_use_the_injected_codec_and_ttl() { Ok("OK"), ), MockCmd::new(redis::cmd("GET").arg("counter"), Ok(vec![42u8, 7])), - ]) - .assert_all_commands_consumed(); - let cache = RedisCache::with_connection(connection, None, TaggedByteCodec(42)); + ]); let context = ExactCacheContext { ttl: Some(Duration::from_millis(1500)), }; @@ -56,34 +95,56 @@ fn generic_helpers_use_the_injected_codec_and_ttl() { assert_eq!(get_cache(&cache, "counter", &context).unwrap(), Some(7)); } -#[tokio::test] -async fn async_operations_preserve_codec_ttl_and_missing_values() { - let connection = MockRedisConnection::new([ +#[rstest] +fn commands_round_trip_entries_and_delete_only_namespaced_keys(context: ExactCacheContext) { + let value = json!({"deployment": "model-a", "cooldown_seconds": 30}); + let payload = JsonCodec::::new() + .encode(&value) + .unwrap(); + let cache = json_cache(vec![ MockCmd::new( redis::cmd("SETEX") - .arg("counter") - .arg(9) - .arg([42u8, 7].as_slice()), + .arg("litellm-cache:key") + .arg(600) + .arg(payload.clone()), Ok("OK"), ), - MockCmd::new(redis::cmd("GET").arg("counter"), Ok(vec![42u8, 7])), - MockCmd::new( - redis::cmd("SETEX") - .arg("batch") - .arg(2) - .arg([42u8, 8].as_slice()), - Ok("OK"), - ), - MockCmd::new(redis::cmd("DEL").arg("counter"), Ok(1u32)), - MockCmd::new(redis::cmd("GET").arg("counter"), Ok(redis::Value::Nil)), + MockCmd::new(redis::cmd("GET").arg("litellm-cache:key"), Ok(payload)), + MockCmd::new(redis::cmd("DEL").arg("litellm-cache:key"), Ok(1u32)), ]) - .assert_all_commands_consumed(); + .with_namespace(Some("litellm-cache".into())); + + cache.set_cache("key", value.clone(), &context).unwrap(); + assert_eq!(cache.get_cache("key", &context).unwrap(), Some(value)); + cache.delete_cache("key").unwrap(); +} + +#[rstest] +#[tokio::test] +async fn async_operations_preserve_codec_ttl_and_missing_values(context: ExactCacheContext) { let cache = RedisCache::with_connection( - connection, + mock(vec![ + MockCmd::new( + redis::cmd("SETEX") + .arg("counter") + .arg(9) + .arg([42u8, 7].as_slice()), + Ok("OK"), + ), + MockCmd::new(redis::cmd("GET").arg("counter"), Ok(vec![42u8, 7])), + MockCmd::new( + redis::cmd("SETEX") + .arg("batch") + .arg(2) + .arg([42u8, 8].as_slice()), + Ok("OK"), + ), + MockCmd::new(redis::cmd("DEL").arg("counter"), Ok(1u32)), + MockCmd::new(redis::cmd("GET").arg("counter"), Ok(redis::Value::Nil)), + ]), Some(Duration::from_secs(9)), TaggedByteCodec(42), ); - let context = ExactCacheContext::default(); cache .batch_cache_write("counter", 7, context.clone()) .await @@ -108,15 +169,13 @@ async fn async_operations_preserve_codec_ttl_and_missing_values() { ); } +#[rstest] #[tokio::test] -async fn codec_errors_propagate_without_writing_partial_batches() { - let connection = MockRedisConnection::new([ +async fn codec_errors_propagate_without_writing_partial_batches(context: ExactCacheContext) { + let cache = tagged(vec![ MockCmd::new(redis::cmd("GET").arg("invalid"), Ok(vec![99u8, 7])), MockCmd::new(redis::cmd("GET").arg("invalid"), Ok(vec![99u8, 7])), - ]) - .assert_all_commands_consumed(); - let cache = RedisCache::with_connection(connection, None, TaggedByteCodec(42)); - let context = ExactCacheContext::default(); + ]); assert_eq!( cache.set_cache("invalid", 255, &context), Err(Error::InvalidEntry) @@ -134,6 +193,15 @@ async fn codec_errors_propagate_without_writing_partial_batches() { .await, Err(Error::InvalidEntry) ); + assert_eq!( + cache + .async_set_cache_pipeline_with_ttls(vec![ + ("valid".into(), 7, None), + ("invalid".into(), 255, None), + ]) + .await, + Err(Error::InvalidEntry) + ); assert_eq!( cache.get_cache("invalid", &context), Err(Error::InvalidEntry) @@ -144,221 +212,269 @@ async fn codec_errors_propagate_without_writing_partial_batches() { ); } -#[test] -fn namespaces_are_optional_and_existing_prefixes_are_not_duplicated() { - let connection = MockRedisConnection::new([ - MockCmd::new(redis::cmd("GET").arg("team:key"), Ok(redis::Value::Nil)), - MockCmd::new(redis::cmd("GET").arg("team:key"), Ok(redis::Value::Nil)), - ]) - .assert_all_commands_consumed(); - let cache = RedisCache::with_connection(connection, None, JsonCodec::::new()) - .with_namespace(Some("team".into())); - assert_eq!( - cache - .get_cache("key", &ExactCacheContext::default()) - .unwrap(), - None - ); - assert_eq!( - cache - .get_cache("team:key", &ExactCacheContext::default()) - .unwrap(), - None - ); +#[rstest] +#[case::bare_key("key")] +#[case::already_prefixed("team:key")] +fn namespaces_are_added_once(#[case] key: &str, context: ExactCacheContext) { + let cache = team(vec![MockCmd::new( + redis::cmd("GET").arg("team:key"), + Ok(redis::Value::Nil), + )]); + assert_eq!(cache.get_cache(key, &context).unwrap(), None); } -#[test] -fn flush_requires_a_namespace_and_escapes_glob_metacharacters() { - let unscoped = RedisCache::with_connection( - MockRedisConnection::new([]).assert_all_commands_consumed(), - None, - JsonCodec::::new(), - ); +#[rstest] +#[case::empty(Some(String::new()))] +#[case::missing(None)] +fn empty_namespaces_leave_keys_unprefixed( + #[case] namespace: Option, + context: ExactCacheContext, +) { + let cache = json_cache(vec![MockCmd::new( + redis::cmd("GET").arg("key"), + Ok(redis::Value::Nil), + )]) + .with_namespace(namespace); + assert_eq!(cache.namespace(), None); + assert_eq!(cache.get_cache("key", &context).unwrap(), None); +} + +#[rstest] +#[tokio::test] +async fn flush_requires_a_namespace() { + let unscoped = json_cache(Vec::new()); assert_eq!(unscoped.flush_cache(), Err(Error::UnscopedFlush)); - let connection = MockRedisConnection::new([ - MockCmd::new( - redis::cmd("SCAN") - .cursor_arg(0) - .arg("MATCH") - .arg("team\\*:*") - .arg("COUNT") - .arg(1000), - Ok(redis_test::redis_value!(["0", ["team*:key"]])), - ), - MockCmd::new(redis::cmd("DEL").arg("team*:key"), Ok(1u32)), + assert_eq!( + unscoped.async_flush_cache().await, + Err(Error::UnscopedFlush) + ); +} + +#[rstest] +#[case::plain_namespace("litellm-cache", "litellm-cache:*", "litellm-cache:key")] +#[case::glob_metacharacters_are_escaped("team*", "team\\*:*", "team*:key")] +fn flush_scans_and_deletes_only_namespaced_keys( + #[case] namespace: &str, + #[case] pattern: &str, + #[case] key: &str, +) { + let cache = json_cache(vec![ + scan(pattern, 0, 1000, redis_test::redis_value!(["0", [key]])), + MockCmd::new(redis::cmd("DEL").arg(key), Ok(1u32)), ]) - .assert_all_commands_consumed(); - let scoped = RedisCache::with_connection(connection, None, JsonCodec::::new()) - .with_namespace(Some("team*".into())); - scoped.flush_cache().unwrap(); + .with_namespace(Some(namespace.into())); + cache.flush_cache().unwrap(); } +#[rstest] #[tokio::test] -async fn connection_failures_use_the_python_result_contract() { - let error = redis::RedisError::from((redis::ErrorKind::Io, "connection refused")); - let connection = - MockRedisConnection::new([MockCmd::new(redis::cmd("PING"), Err::(error))]) - .assert_all_commands_consumed(); - let cache = RedisCache::with_connection(connection, None, JsonCodec::::new()); - - let result = cache.test_connection().await.unwrap(); - assert_eq!(result.status, CacheConnectionStatus::Failed); - assert!(result.message.starts_with("Redis connection failed:")); - assert!(result.error.is_some()); +async fn async_flush_deletes_each_scan_page_separately() { + let cache = team(vec![ + scan( + "team:*", + 0, + 1000, + redis_test::redis_value!(["7", ["team:a", "team:b"]]), + ), + MockCmd::new(redis::cmd("DEL").arg("team:a").arg("team:b"), Ok(2u32)), + scan( + "team:*", + 7, + 1000, + redis_test::redis_value!(["0", ["team:c"]]), + ), + MockCmd::new(redis::cmd("DEL").arg("team:c"), Ok(1u32)), + ]); + cache.async_flush_cache().await.unwrap(); } +#[rstest] +fn flushall_ignores_the_namespace() { + team(vec![MockCmd::new(redis::cmd("FLUSHALL"), Ok("OK"))]) + .flushall() + .unwrap(); +} + +#[rstest] #[tokio::test] -async fn batch_reads_keep_order_and_treat_invalid_values_as_invalid_entries() { - let connection = MockRedisConnection::new([MockCmd::new( +async fn batch_reads_keep_order_and_treat_invalid_values_as_invalid_entries( + context: ExactCacheContext, +) { + let cache = tagged(vec![MockCmd::new( redis::cmd("MGET").arg("hit").arg("miss").arg("invalid"), Ok(vec![ redis::Value::BulkString(vec![42, 7]), redis::Value::Nil, redis::Value::BulkString(vec![99, 7]), ]), - )]) - .assert_all_commands_consumed(); - let cache = RedisCache::with_connection(connection, None, TaggedByteCodec(42)); + )]); assert_eq!( cache - .async_batch_get_cache( - vec!["hit".into(), "miss".into(), "invalid".into()], - ExactCacheContext::default(), - ) + .async_batch_get_cache(vec!["hit".into(), "miss".into(), "invalid".into()], context) .await .unwrap(), vec![BatchEntry::Hit(7), BatchEntry::Miss, BatchEntry::Invalid] ); } +#[rstest] #[tokio::test] -async fn async_flush_deletes_each_scan_page_separately() { - let connection = MockRedisConnection::new([ - MockCmd::new( - redis::cmd("SCAN") - .cursor_arg(0) - .arg("MATCH") - .arg("team:*") - .arg("COUNT") - .arg(1000), - Ok(redis_test::redis_value!(["7", ["team:a", "team:b"]])), - ), - MockCmd::new(redis::cmd("DEL").arg("team:a").arg("team:b"), Ok(2u32)), - MockCmd::new( - redis::cmd("SCAN") - .cursor_arg(7) - .arg("MATCH") - .arg("team:*") - .arg("COUNT") - .arg(1000), - Ok(redis_test::redis_value!(["0", ["team:c"]])), - ), - MockCmd::new(redis::cmd("DEL").arg("team:c"), Ok(1u32)), - ]) - .assert_all_commands_consumed(); - let cache = RedisCache::with_connection(connection, None, JsonCodec::::new()) - .with_namespace(Some("team".into())); +async fn ttl_pipeline_keeps_each_entry_ttl_and_defaults_missing_ones() { + let mut pipeline = redis::pipe(); + pipeline + .cmd("SETEX") + .arg("ns:team_id:t1") + .arg(60u64) + .arg(r#"{"team_id":"t1"}"#) + .cmd("SETEX") + .arg("ns:u1") + .arg(7u64) + .arg(r#"{"user_id":"u1"}"#) + .cmd("SETEX") + .arg("ns:org_id:o1") + .arg(300u64) + .arg(r#"{"a":1}"#); + let cache = RedisCache::with_connection( + mock(vec![MockCmd::with_values( + pipeline, + Ok(vec!["OK", "OK", "OK"]), + )]), + Some(Duration::from_secs(300)), + JsonCodec::::new(), + ) + .with_namespace(Some("ns".into())); - cache.async_flush_cache().await.unwrap(); + cache + .async_set_cache_pipeline_with_ttls(vec![ + ( + "team_id:t1".into(), + json!({"team_id": "t1"}), + Some(Duration::from_secs(60)), + ), + ( + "u1".into(), + json!({"user_id": "u1"}), + Some(Duration::from_secs(7)), + ), + ("org_id:o1".into(), json!({"a": 1}), None), + ]) + .await + .unwrap(); } +#[rstest] #[tokio::test] -async fn direct_redis_operations_preserve_namespace_values_and_missing_ttls() { - let mut sadd_pipeline = redis::pipe(); - sadd_pipeline - .cmd("SADD") - .arg("team:members") - .arg("a") - .arg("b") - .cmd("EXPIRE") - .arg("team:members") - .arg(600u64) - .ignore(); - let connection = MockRedisConnection::new([ +async fn empty_pipelines_skip_the_round_trip(context: ExactCacheContext) { + let cache = json_cache(Vec::new()); + cache + .async_set_cache_pipeline(Vec::new(), context) + .await + .unwrap(); + cache + .async_set_cache_pipeline_with_ttls(Vec::new()) + .await + .unwrap(); + assert_eq!(cache.delete_cache_keys(Vec::new()).await.unwrap(), 0); + assert_eq!( + cache.async_rpush_pipeline(Vec::new()).await.unwrap(), + Vec::::new() + ); + assert_eq!( + cache.async_lpop_pipeline(Vec::new()).await.unwrap(), + Vec::::new() + ); + assert_eq!( + cache.async_increment_pipeline(Vec::new()).await.unwrap(), + Vec::::new() + ); +} + +#[rstest] +#[tokio::test] +async fn count_reads_parse_integers_and_keep_missing_counters() { + let mget = || { MockCmd::new( redis::cmd("MGET").arg("team:count").arg("team:missing"), Ok(redis_test::redis_value!(["7", nil])), - ), - MockCmd::new( - redis::cmd("MGET").arg("team:count").arg("team:missing"), - Ok(redis_test::redis_value!(["7", nil])), - ), + ) + }; + let cache = team(vec![mget(), mget()]); + let keys = vec!["count".to_string(), "missing".to_string()]; + + assert_eq!(cache.batch_get_counts(&keys).unwrap(), [Some(7), None]); + assert_eq!( + cache.async_batch_get_counts(keys).await.unwrap(), + [Some(7), None] + ); +} + +#[rstest] +#[tokio::test] +async fn pings_run_on_both_paths() { + let cache = team(vec![ MockCmd::new(redis::cmd("PING"), Ok("PONG")), MockCmd::new(redis::cmd("PING"), Ok("PONG")), - MockCmd::new(redis::cmd("TTL").arg("team:missing"), Ok(-2i64)), - MockCmd::new( - redis::cmd("SCAN") - .cursor_arg(0) - .arg("MATCH") - .arg("team:job-*") - .arg("COUNT") - .arg(25), - Ok(redis_test::redis_value!(["4", ["team:job-a"]])), + ]); + assert!(cache.sync_ping().unwrap()); + assert!(cache.ping().await.unwrap()); +} + +#[rstest] +#[case::remaining(12, Some(Duration::from_secs(12)))] +#[case::no_expiry(-1, None)] +#[case::missing(-2, None)] +#[tokio::test] +async fn ttl_reads_hide_negative_replies(#[case] reply: i64, #[case] ttl: Option) { + let cache = team(vec![MockCmd::new( + redis::cmd("TTL").arg("team:key"), + Ok(reply), + )]); + assert_eq!(cache.async_get_ttl("key").await.unwrap(), ttl); +} + +#[rstest] +#[case::explicit_ttl(Some(Duration::from_secs(30)), 30, 1, true)] +#[case::default_ttl(None, 600, 1, true)] +#[case::missing_key(Some(Duration::from_secs(30)), 30, 0, false)] +#[tokio::test] +async fn refresh_ttl_expires_existing_keys_only( + #[case] ttl: Option, + #[case] seconds: u64, + #[case] reply: i64, + #[case] refreshed: bool, +) { + let cache = team(vec![MockCmd::new( + redis::cmd("EXPIRE").arg("team:key").arg(seconds), + Ok(reply), + )]); + assert_eq!( + cache.async_refresh_ttl("key", ttl).await.unwrap(), + refreshed + ); +} + +#[rstest] +#[tokio::test] +async fn scan_stops_at_count_and_bulk_delete_reports_existing_keys() { + let cache = team(vec![ + scan( + "team:job-*", + 0, + 25, + redis_test::redis_value!(["4", ["team:job-a"]]), ), - MockCmd::new( - redis::cmd("SCAN") - .cursor_arg(4) - .arg("MATCH") - .arg("team:job-*") - .arg("COUNT") - .arg(25), - Ok(redis_test::redis_value!(["0", ["team:job-b"]])), + scan( + "team:job-*", + 4, + 25, + redis_test::redis_value!(["0", ["team:job-b"]]), ), MockCmd::new( redis::cmd("DEL").arg("team:job-a").arg("team:job-b"), Ok(2u32), ), - MockCmd::with_values( - sadd_pipeline, - Ok(vec![redis::Value::Int(2), redis::Value::Int(1)]), - ), - MockCmd::new( - redis::cmd("RPUSH").arg("team:queue").arg("a").arg("b"), - Ok(2u32), - ), - MockCmd::new( - redis::cmd("LPOP").arg("team:queue").arg(2usize), - Ok(redis_test::redis_value!(["a", "b"])), - ), - MockCmd::new( - redis::cmd("EVAL") - .arg("return KEYS[1]") - .arg(1usize) - .arg("team:key"), - Ok("team:key"), - ), - MockCmd::new( - redis::cmd("EVAL") - .arg("return KEYS[1]") - .arg(1usize) - .arg("team:key"), - Ok("team:key"), - ), - MockCmd::new(redis::cmd("CLIENT").arg("LIST"), Ok("id=1")), - MockCmd::new(redis::cmd("INFO"), Ok("redis_version:7")), - MockCmd::new(redis::cmd("FLUSHALL"), Ok("OK")), - ]) - .assert_all_commands_consumed(); - let cache = RedisCache::with_connection(connection, None, JsonCodec::::new()) - .with_namespace(Some("team".into())); - - assert_eq!( - cache - .batch_get_counts(&["count".into(), "missing".into()]) - .unwrap(), - [Some(7), None] - ); - assert_eq!( - cache - .async_batch_get_counts(vec!["count".into(), "missing".into()]) - .await - .unwrap(), - [Some(7), None] - ); - assert!(cache.sync_ping().unwrap()); - assert!(cache.ping().await.unwrap()); - assert_eq!(cache.async_get_ttl("missing").await.unwrap(), None); + ]); assert_eq!( cache.async_scan_iter("job-", 25).await.unwrap(), ["team:job-a", "team:job-b"] @@ -370,6 +486,25 @@ async fn direct_redis_operations_preserve_namespace_values_and_missing_ttls() { .unwrap(), 2 ); +} + +#[rstest] +#[tokio::test] +async fn sets_add_members_and_arm_the_default_ttl() { + let mut pipeline = redis::pipe(); + pipeline + .cmd("SADD") + .arg("team:members") + .arg("a") + .arg("b") + .cmd("EXPIRE") + .arg("team:members") + .arg(600u64) + .ignore(); + let cache = team(vec![MockCmd::with_values( + pipeline, + Ok(vec![redis::Value::Int(2), redis::Value::Int(1)]), + )]); assert_eq!( cache .async_set_cache_sadd("members", vec!["a".into(), "b".into()], None) @@ -377,6 +512,32 @@ async fn direct_redis_operations_preserve_namespace_values_and_missing_ttls() { .unwrap(), 2 ); + assert_eq!( + cache + .async_set_cache_sadd("members", Vec::new(), None) + .await, + Err(Error::InvalidEntry) + ); +} + +#[rstest] +#[tokio::test] +async fn queues_push_and_pop_namespaced_lists() { + let cache = team(vec![ + MockCmd::new( + redis::cmd("RPUSH").arg("team:queue").arg("a").arg("b"), + Ok(2u32), + ), + MockCmd::new( + redis::cmd("INFO"), + Ok("# Server\r\nredis_version:7.2.4\r\n"), + ), + MockCmd::new( + redis::cmd("LPOP").arg("team:queue").arg(2usize), + Ok(redis_test::redis_value!(["a", "b"])), + ), + MockCmd::new(redis::cmd("LPOP").arg("team:queue"), Ok("c")), + ]); assert_eq!( cache .async_rpush("queue", vec!["a".into(), "b".into()]) @@ -384,10 +545,159 @@ async fn direct_redis_operations_preserve_namespace_values_and_missing_ttls() { .unwrap(), 2 ); + assert_eq!( + cache.async_rpush("queue", Vec::new()).await, + Err(Error::InvalidEntry) + ); assert_eq!( cache.async_lpop("queue", Some(2)).await.unwrap(), RedisLpopResult::Values(vec![b"a".to_vec(), b"b".to_vec()]) ); + assert_eq!( + cache.async_lpop("queue", None).await.unwrap(), + RedisLpopResult::Value(b"c".to_vec()) + ); +} + +/// Python `RedisCache.async_lpop` checks `redis_version` from `INFO` and, below major version 7, +/// pops a counted batch as `count` single-command `LPOP` pipelines, dropping `None` replies. +#[rstest] +#[tokio::test] +async fn counted_lpop_falls_back_to_single_pops_below_redis_7() { + let single_pop = || { + let mut pipeline = redis::pipe(); + pipeline.cmd("LPOP").arg("team:queue"); + pipeline + }; + let cache = team(vec![ + MockCmd::new( + redis::cmd("INFO"), + Ok("# Server\r\nredis_version:6.2.14\r\n"), + ), + MockCmd::with_values(single_pop(), Ok(vec![redis_test::redis_value!("a")])), + MockCmd::with_values(single_pop(), Ok(vec![redis_test::redis_value!("b")])), + MockCmd::with_values(single_pop(), Ok(vec![redis::Value::Nil])), + ]); + + assert_eq!( + cache.async_lpop("queue", Some(3)).await.unwrap(), + RedisLpopResult::Values(vec![b"a".to_vec(), b"b".to_vec()]) + ); +} + +/// Python keeps `redis_version = "Unknown"` when `INFO` fails and then assumes +/// `DEFAULT_REDIS_MAJOR_VERSION` (7), so a counted pop is one `LPOP key count`. The version is read +/// once: the second pop sends no second `INFO`. +#[rstest] +#[tokio::test] +async fn counted_lpop_assumes_redis_7_when_info_fails() { + let cache = team(vec![ + MockCmd::new( + redis::cmd("INFO"), + Err::(redis::RedisError::from((redis::ErrorKind::Io, "down"))), + ), + MockCmd::new( + redis::cmd("LPOP").arg("team:queue").arg(2usize), + Ok(redis_test::redis_value!(["a", "b"])), + ), + MockCmd::new( + redis::cmd("LPOP").arg("team:queue").arg(1usize), + Ok(redis_test::redis_value!(["c"])), + ), + ]); + + assert_eq!( + cache.async_lpop("queue", Some(2)).await.unwrap(), + RedisLpopResult::Values(vec![b"a".to_vec(), b"b".to_vec()]) + ); + assert_eq!( + cache.async_lpop("queue", Some(1)).await.unwrap(), + RedisLpopResult::Values(vec![b"c".to_vec()]) + ); +} + +fn push_and_trim(start: i64) -> redis::Pipeline { + let mut pipeline = redis::pipe(); + pipeline + .atomic() + .cmd("RPUSH") + .arg("ns:buf") + .arg("c") + .arg("d") + .cmd("LTRIM") + .arg("ns:buf") + .arg(start) + .arg(-1); + pipeline +} + +#[rstest] +#[case::keeps_newest_entries(3, -3)] +#[case::zero_keeps_everything(0, 0)] +#[tokio::test] +async fn rpush_and_trim_runs_push_and_trim_in_one_transaction( + #[case] max_len: usize, + #[case] start: i64, +) { + let cache = json_cache(vec![MockCmd::with_values( + push_and_trim(start), + Ok(vec![redis::Value::Array(vec![ + redis::Value::Int(4), + redis::Value::Okay, + ])]), + )]) + .with_namespace(Some("ns".into())); + + assert_eq!( + cache + .async_rpush_and_trim("buf", vec!["c".into(), "d".into()], max_len) + .await + .unwrap(), + 4 + ); +} + +#[rstest] +#[tokio::test] +async fn rpush_and_trim_raises_when_a_queued_command_fails() { + let wrong_type = redis::parse_redis_value( + b"-WRONGTYPE Operation against a key holding the wrong kind of value\r\n", + ) + .unwrap(); + let cache = json_cache(vec![MockCmd::with_values( + push_and_trim(-3), + Ok(vec![redis::Value::Array(vec![ + wrong_type, + redis::Value::Okay, + ])]), + )]) + .with_namespace(Some("ns".into())); + + assert_eq!( + cache + .async_rpush_and_trim("buf", vec!["c".into(), "d".into()], 3) + .await, + Err(Error::Unavailable) + ); + assert_eq!( + cache.async_rpush_and_trim("buf", Vec::new(), 3).await, + Err(Error::InvalidEntry) + ); +} + +#[rstest] +#[tokio::test] +async fn scripts_and_eval_namespace_their_keys() { + let eval = || { + MockCmd::new( + redis::cmd("EVAL") + .arg("return KEYS[1]") + .arg(1usize) + .arg("team:key"), + Ok("team:key"), + ) + }; + let cache = team(vec![eval(), eval()]); assert_eq!( cache .async_eval("return KEYS[1]".into(), vec!["key".into()], Vec::new()) @@ -403,13 +713,21 @@ async fn direct_redis_operations_preserve_namespace_values_and_missing_ttls() { .unwrap(), redis::Value::BulkString(b"team:key".to_vec()) ); - assert_eq!(cache.client_list().unwrap(), "id=1"); - assert_eq!(cache.info().unwrap(), "redis_version:7"); - cache.flushall().unwrap(); } +#[rstest] +fn client_list_and_info_return_server_text() { + let cache = team(vec![ + MockCmd::new(redis::cmd("CLIENT").arg("LIST"), Ok("id=1")), + MockCmd::new(redis::cmd("INFO"), Ok("redis_version:7")), + ]); + assert_eq!(cache.client_list().unwrap(), "id=1"); + assert_eq!(cache.info().unwrap(), "redis_version:7"); +} + +#[rstest] #[tokio::test] -async fn direct_redis_pipelines_preserve_operation_order() { +async fn pipelines_preserve_operation_order() { let mut rpush_pipeline = redis::pipe(); rpush_pipeline .cmd("RPUSH") @@ -425,19 +743,20 @@ async fn direct_redis_pipelines_preserve_operation_order() { .arg(2usize) .cmd("LPOP") .arg("team:b"); - let connection = MockRedisConnection::new([ + let queue = team(vec![ MockCmd::with_values( rpush_pipeline, Ok(vec![redis::Value::Int(1), redis::Value::Int(2)]), ), + MockCmd::new( + redis::cmd("INFO"), + Ok("# Server\r\nredis_version:7.2.4\r\n"), + ), MockCmd::with_values( lpop_pipeline, Ok(vec![redis_test::redis_value!(["one"]), redis::Value::Nil]), ), - ]) - .assert_all_commands_consumed(); - let queue = RedisCache::with_connection(connection, None, JsonCodec::::new()) - .with_namespace(Some("team".into())); + ]); assert_eq!( queue @@ -474,7 +793,59 @@ async fn direct_redis_pipelines_preserve_operation_order() { RedisLpopResult::Missing, ] ); +} +/// Below major version 7 a counted `LPOP` is unsupported, so a pipeline that mixes counted and +/// plain pops runs each operation through `async_lpop`: `count` single-`LPOP` pipelines for the +/// counted ones, a bare `LPOP` for the rest. No `LPOP key count` reaches the connection. +#[rstest] +#[tokio::test] +async fn lpop_pipeline_pops_one_at_a_time_below_redis_7() { + let single_pop = |key: &str| { + let mut pipeline = redis::pipe(); + pipeline.cmd("LPOP").arg(key); + pipeline + }; + let cache = team(vec![ + MockCmd::new( + redis::cmd("INFO"), + Ok("# Server\r\nredis_version:6.2.14\r\n"), + ), + MockCmd::with_values( + single_pop("team:a"), + Ok(vec![redis_test::redis_value!("one")]), + ), + MockCmd::with_values( + single_pop("team:a"), + Ok(vec![redis_test::redis_value!("two")]), + ), + MockCmd::new(redis::cmd("LPOP").arg("team:b"), Ok(redis::Value::Nil)), + ]); + + assert_eq!( + cache + .async_lpop_pipeline(vec![ + RedisLpopOperation { + key: "a".into(), + count: Some(2), + }, + RedisLpopOperation { + key: "b".into(), + count: None, + }, + ]) + .await + .unwrap(), + [ + RedisLpopResult::Values(vec![b"one".to_vec(), b"two".to_vec()]), + RedisLpopResult::Missing, + ] + ); +} + +#[rstest] +#[tokio::test] +async fn increment_pipeline_expires_only_operations_with_a_ttl() { let mut increment_pipeline = redis::pipe(); increment_pipeline .cmd("INCRBYFLOAT") @@ -487,17 +858,14 @@ async fn direct_redis_pipelines_preserve_operation_order() { .cmd("INCRBYFLOAT") .arg("team:counter") .arg(2.0f64); - let connection = MockRedisConnection::new([MockCmd::with_values( + let counters = team(vec![MockCmd::with_values( increment_pipeline, Ok(vec![ redis::Value::BulkString(b"1.5".to_vec()), redis::Value::Int(1), redis::Value::BulkString(b"3.5".to_vec()), ]), - )]) - .assert_all_commands_consumed(); - let counters = RedisCache::with_connection(connection, None, JsonCodec::::new()) - .with_namespace(Some("team".into())); + )]); assert_eq!( counters .async_increment_pipeline(vec![ @@ -518,6 +886,11 @@ async fn direct_redis_pipelines_preserve_operation_order() { ); } +const INCREMENT_SCRIPT: &str = concat!( + "local value = redis.call('INCRBYFLOAT', KEYS[1], ARGV[1]); ", + "if redis.call('TTL', KEYS[1]) == -1 then ", + "redis.call('EXPIRE', KEYS[1], ARGV[2]); end; return value" +); const INCREMENT_WITH_FLOOR_SCRIPT: &str = concat!( "local count = redis.call('INCRBY', KEYS[1], ARGV[1]); ", "if count < 0 then count = redis.call('INCRBY', KEYS[1], -count); end; ", @@ -532,20 +905,93 @@ const SET_MAX_SCRIPT: &str = concat!( "return ARGV[1]; end; return current" ); +fn increment_script(amount: f64) -> MockCmd { + MockCmd::new( + redis::cmd("EVAL") + .arg(INCREMENT_SCRIPT) + .arg(1) + .arg("counter") + .arg(amount) + .arg(600), + Ok("4.5"), + ) +} + +#[rstest] +#[tokio::test] +async fn increments_keep_an_existing_ttl_in_one_atomic_script(context: ExactCacheContext) { + let cache = RedisCache::with_connection( + mock(vec![increment_script(2.5), increment_script(2.5)]), + None, + JsonCodec::::new(), + ); + + assert_eq!( + cache + .increment_cache("counter", 2.5, context.clone()) + .unwrap(), + 4.5 + ); + assert_eq!( + cache + .async_increment("counter", 2.5, context, false) + .await + .unwrap(), + 4.5 + ); +} + +#[rstest] +#[case::explicit_ttl(Some(Duration::from_secs(60)), 60u64)] +#[case::default_ttl(None, 600u64)] +#[tokio::test] +async fn refresh_ttl_increments_rearm_the_ttl_in_the_same_round_trip( + #[case] ttl: Option, + #[case] seconds: u64, +) { + let mut pipeline = redis::pipe(); + pipeline + .cmd("INCRBYFLOAT") + .arg("ns:spend:key:k") + .arg(1.5f64) + .cmd("EXPIRE") + .arg("ns:spend:key:k") + .arg(seconds); + let cache = json_cache(vec![MockCmd::with_values( + pipeline, + Ok(vec![ + redis::Value::BulkString(b"1.5".to_vec()), + redis::Value::Int(1), + ]), + )]) + .with_namespace(Some("ns".into())); + + assert_eq!( + cache + .async_increment("spend:key:k", 1.5, ExactCacheContext { ttl }, true) + .await + .unwrap(), + 1.5 + ); +} + +#[rstest] #[tokio::test] async fn counter_repairs_are_atomic_and_use_default_ttl() { let floor = || { - redis::cmd("EVAL") - .arg(INCREMENT_WITH_FLOOR_SCRIPT) - .arg(1) - .arg("team:counter") - .arg(-2i64) - .arg(30u64) - .clone() + MockCmd::new( + redis::cmd("EVAL") + .arg(INCREMENT_WITH_FLOOR_SCRIPT) + .arg(1) + .arg("team:counter") + .arg(-2i64) + .arg(30u64), + Ok(0i64), + ) }; - let connection = MockRedisConnection::new([ - MockCmd::new(floor(), Ok(0i64)), - MockCmd::new(floor(), Ok(0i64)), + let cache = team(vec![ + floor(), + floor(), MockCmd::new( redis::cmd("EVAL") .arg(SET_MAX_SCRIPT) @@ -555,10 +1001,7 @@ async fn counter_repairs_are_atomic_and_use_default_ttl() { .arg(600u64), Ok("4.5"), ), - ]) - .assert_all_commands_consumed(); - let cache = RedisCache::with_connection(connection, None, JsonCodec::::new()) - .with_namespace(Some("team".into())); + ]); assert_eq!( cache @@ -587,38 +1030,37 @@ const CLAIM_SCRIPT: &str = concat!( "elseif ARGV[4] == '1' then redis.call('EXPIRE', KEYS[1], ARGV[2]); end; return 1" ); -fn claim_eval(expected: &str, write: &str, refresh: bool) -> redis::Cmd { - let mut cmd = redis::cmd("EVAL"); - cmd.arg(CLAIM_SCRIPT) - .arg(1) - .arg("pin") - .arg(expected) - .arg(600) - .arg(write) - .arg(u8::from(refresh)); - cmd +fn claim_eval(expected: &str, write: &str, refresh: bool, applied: i64) -> MockCmd { + MockCmd::new( + redis::cmd("EVAL") + .arg(CLAIM_SCRIPT) + .arg(1) + .arg("pin") + .arg(expected) + .arg(600) + .arg(write) + .arg(u8::from(refresh)), + Ok(applied), + ) } +#[rstest] #[tokio::test] -async fn claims_match_eligible_values_written_by_another_encoder() { +async fn claims_match_eligible_values_written_by_another_encoder(context: ExactCacheContext) { let python_payload = r#"{"model_id": "a", "deployment": "east"}"#; - let stored = serde_json::json!({"deployment": "east", "model_id": "a"}); - let candidate = serde_json::json!({"model_id": "b"}); - let connection = MockRedisConnection::new([ + let stored = json!({"deployment": "east", "model_id": "a"}); + let cache = json_cache(vec![ MockCmd::new(redis::cmd("GET").arg("pin"), Ok(python_payload)), - MockCmd::new(claim_eval(python_payload, "", true), Ok(1)), - ]) - .assert_all_commands_consumed(); - let cache = - RedisCache::with_connection(connection, None, JsonCodec::::new()); + claim_eval(python_payload, "", true, 1), + ]); assert_eq!( cache .async_claim_cache( "pin", - candidate, + json!({"model_id": "b"}), vec![stored.clone()], - ExactCacheContext::default() + context ) .await .unwrap(), @@ -626,78 +1068,91 @@ async fn claims_match_eligible_values_written_by_another_encoder() { ); } -#[test] -fn claims_retry_when_the_key_changes_and_replace_ineligible_winners() { - let candidate = serde_json::json!({"model_id": "b"}); +#[rstest] +fn claims_retry_when_the_key_changes_and_replace_ineligible_winners(context: ExactCacheContext) { + let candidate = json!({"model_id": "b"}); let payload = r#"{"model_id":"b"}"#; - let connection = MockRedisConnection::new([ + let cache = json_cache(vec![ MockCmd::new(redis::cmd("GET").arg("pin"), Ok(redis::Value::Nil)), - MockCmd::new(claim_eval("", payload, false), Ok(0)), + claim_eval("", payload, false, 0), MockCmd::new(redis::cmd("GET").arg("pin"), Ok(r#"{"model_id":"gone"}"#)), - MockCmd::new(claim_eval(r#"{"model_id":"gone"}"#, payload, false), Ok(1)), - ]) - .assert_all_commands_consumed(); - let cache = - RedisCache::with_connection(connection, None, JsonCodec::::new()); + claim_eval(r#"{"model_id":"gone"}"#, payload, false, 1), + ]); assert_eq!( cache .claim_cache( "pin", candidate.clone(), - &[serde_json::json!({"model_id": "a"})], - ExactCacheContext::default() + &[json!({"model_id": "a"})], + context ) .unwrap(), candidate ); } -#[test] -fn claims_without_eligible_values_keep_the_winner_without_refreshing_its_ttl() { +#[rstest] +fn claims_without_eligible_values_keep_the_winner_without_refreshing_its_ttl( + context: ExactCacheContext, +) { let stored = r#"{"model_id": "a"}"#; - let connection = MockRedisConnection::new([ + let cache = json_cache(vec![ MockCmd::new(redis::cmd("GET").arg("pin"), Ok(stored)), - MockCmd::new(claim_eval(stored, "", false), Ok(1)), - ]) - .assert_all_commands_consumed(); - let cache = - RedisCache::with_connection(connection, None, JsonCodec::::new()); + claim_eval(stored, "", false, 1), + ]); assert_eq!( cache - .claim_cache( - "pin", - serde_json::json!({"model_id": "b"}), - &[], - ExactCacheContext::default() - ) + .claim_cache("pin", json!({"model_id": "b"}), &[], context) .unwrap(), - serde_json::json!({"model_id": "a"}) + json!({"model_id": "a"}) ); } +#[rstest] #[tokio::test] -async fn async_increment_runs_the_atomic_script() { - let mut eval = redis::cmd("EVAL"); - eval.arg(concat!( - "local value = redis.call('INCRBYFLOAT', KEYS[1], ARGV[1]); ", - "if redis.call('TTL', KEYS[1]) == -1 then ", - "redis.call('EXPIRE', KEYS[1], ARGV[2]); end; return value" - )) - .arg(1) - .arg("counter") - .arg(2.5f64) - .arg(600); - let connection = - MockRedisConnection::new([MockCmd::new(eval, Ok("4.5"))]).assert_all_commands_consumed(); - let cache = RedisCache::with_connection(connection, None, JsonCodec::::new()); +async fn test_connection_reports_success_with_the_python_message() { + let cache = team(vec![MockCmd::new(redis::cmd("PING"), Ok("PONG"))]); - assert_eq!( - cache - .async_increment("counter", 2.5, ExactCacheContext::default()) - .await - .unwrap(), - 4.5 - ); + let result = cache.test_connection().await.unwrap(); + assert_eq!(result.status, CacheConnectionStatus::Success); + assert_eq!(result.message, "Redis connection test successful"); + assert_eq!(result.error, None); +} + +#[rstest] +#[case::unexpected_reply(Ok("NOPE"), "Redis ping returned False", false)] +#[case::connection_refused( + Err(redis::RedisError::from((redis::ErrorKind::Io, "connection refused"))), + "Redis connection failed:", + true +)] +#[tokio::test] +async fn test_connection_failures_use_the_python_result_contract( + #[case] reply: redis::RedisResult<&'static str>, + #[case] message: &str, + #[case] has_error: bool, +) { + let cache = json_cache(vec![MockCmd::new(redis::cmd("PING"), reply)]); + + let result = cache.test_connection().await.unwrap(); + assert_eq!(result.status, CacheConnectionStatus::Failed); + assert!(result.message.starts_with(message), "{}", result.message); + assert_eq!(result.error.is_some(), has_error); +} + +#[rstest] +#[tokio::test] +async fn disconnect_keeps_a_caller_owned_connection_usable() { + let cache = team(vec![MockCmd::new(redis::cmd("PING"), Ok("PONG"))]); + cache.disconnect().await.unwrap(); + assert!(cache.ping().await.unwrap()); +} + +#[rstest] +#[tokio::test] +async fn disconnect_drains_an_idle_pool_without_connecting() { + let cache = RedisCache::new("redis://127.0.0.1:1", None, JsonCodec::::new()).unwrap(); + cache.disconnect().await.unwrap(); } diff --git a/litellm-rust/crates/cache-redis/tests/cluster.rs b/litellm-rust/crates/cache-redis/tests/cluster.rs index 2c3fc818b66..c1a9a70a5d2 100644 --- a/litellm-rust/crates/cache-redis/tests/cluster.rs +++ b/litellm-rust/crates/cache-redis/tests/cluster.rs @@ -1,100 +1,67 @@ -//! Contract tests against a real Redis Cluster. Set `LITELLM_TEST_REDIS_CLUSTER_NODES` to a -//! comma separated `host:port` list (for example `127.0.0.1:7000,127.0.0.1:7001`) to run them. +//! Tests against a real Redis Cluster. Set `LITELLM_TEST_REDIS_CLUSTER_NODES` to a comma +//! separated `host:port` list (for example `127.0.0.1:7000,127.0.0.1:7001`) to run them. -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +mod support; + +use std::{collections::HashSet, time::Duration}; use litellm_cache::{ - BaseCache, BatchCache, BatchEntry, CacheConnectionStatus, CacheScript, ClaimCache, - CounterCache, DeleteCache, Error, ExactCacheContext, FlushCache, IncrementOperation, JsonCodec, - ScriptCache, + BaseCache, BatchCache, BatchEntry, BoundedCounterCache, BulkDeleteCache, CacheConnectionStatus, + CacheScript, ClaimCache, ClientInfoCache, ConnectionCache, CounterCache, DeleteCache, + DisconnectCache, Error, ExactCacheContext, FlushCache, IncrementOperation, JsonCodec, + PingCache, QueueCache, RefreshTtlCache, ScanCache, ScriptCache, SetCache, TtlCache, + TtlPipelineCache, }; use litellm_cache_redis::{ RedisArg, RedisCache, RedisLpopOperation, RedisLpopResult, RedisNode, RedisRpushOperation, RedisTopology, }; use redis::cluster_routing::Slot; +use rstest::{fixture, rstest}; +use serde_json::json; +use support::{JsonCache, cluster_cache, cluster_url}; -type Cache = RedisCache>; +type Counter = RedisCache>; -fn topology() -> Option { - let nodes = std::env::var("LITELLM_TEST_REDIS_CLUSTER_NODES").ok()?; - let startup_nodes = nodes - .split(',') - .map(|node| { - let (host, port) = node.trim().rsplit_once(':').expect("host:port"); - RedisNode { - host: host.to_string(), - port: port.parse().expect("port"), - } - }) - .collect(); - Some(RedisTopology::Cluster { startup_nodes }) +#[fixture] +fn cache(#[default("cache")] label: &str) -> Option { + cluster_cache(label, Duration::from_secs(120), JsonCodec::new()) } -fn namespace(label: &str) -> String { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos(); - format!("cluster-test:{label}:{nanos}") +#[fixture] +fn counter(#[default("counter")] label: &str) -> Option { + cluster_cache(label, Duration::from_secs(60), JsonCodec::new()) } -fn cluster_url() -> String { - std::env::var("LITELLM_TEST_REDIS_CLUSTER_URL") - .unwrap_or_else(|_| "redis://127.0.0.1:7000".into()) -} - -fn cluster_cache(label: &str) -> Option { - let topology = topology()?; - Some( - Cache::connect( - &cluster_url(), - &topology, - Some(Duration::from_secs(120)), - JsonCodec::new(), - ) - .expect("cluster connection") - .with_namespace(Some(namespace(label))), - ) -} - -fn counter_cache(label: &str) -> Option>> { - let topology = topology()?; - Some( - RedisCache::connect( - &cluster_url(), - &topology, - Some(Duration::from_secs(60)), - JsonCodec::new(), - ) - .expect("cluster connection") - .with_namespace(Some(namespace(label))), - ) +#[fixture] +fn context() -> ExactCacheContext { + ExactCacheContext::default() } fn multi_slot_keys(count: usize) -> Vec { let keys: Vec = (0..count).map(|index| format!("key-{index}")).collect(); - let slots: std::collections::HashSet = keys.iter().map(Slot::for_key).collect(); + let slots: HashSet = keys.iter().map(Slot::for_key).collect(); assert!(slots.len() > 1, "keys must span multiple slots"); keys } -macro_rules! cluster_or_skip { - ($label:expr) => { - match cluster_cache($label) { - Some(cache) => cache, - None => return, - } - }; +fn seconds(seconds: u64) -> Option { + Some(Duration::from_secs(seconds)) } -#[test] -fn constructor_rejects_clusters_without_startup_nodes() { - let error = Cache::connect( - "redis://127.0.0.1:7000", - &RedisTopology::Cluster { - startup_nodes: Vec::new(), - }, +#[rstest] +#[case::no_startup_nodes("redis://127.0.0.1:7000", Vec::new())] +#[case::unix_socket_url( + "redis+unix:///tmp/redis.sock", + vec![RedisNode { host: "127.0.0.1".into(), port: 7000 }] +)] +fn constructor_rejects_unusable_cluster_configs( + #[case] url: &str, + #[case] startup_nodes: Vec, +) { + let error = JsonCache::connect( + url, + &RedisTopology::Cluster { startup_nodes }, None, JsonCodec::new(), ) @@ -102,60 +69,56 @@ fn constructor_rejects_clusters_without_startup_nodes() { assert!(matches!(error, Some(Error::Unavailable))); } -#[test] -fn constructor_rejects_unix_socket_urls_for_clusters() { - let error = Cache::connect( - "redis+unix:///tmp/redis.sock", - &RedisTopology::Cluster { - startup_nodes: vec![RedisNode { - host: "127.0.0.1".into(), - port: 7000, - }], - }, - None, - JsonCodec::new(), - ) - .err(); - assert!(matches!(error, Some(Error::Unavailable))); -} - -#[test] -fn single_key_operations_round_trip_with_ttl_rounding() { - let cache = cluster_or_skip!("single"); +#[rstest] +#[tokio::test] +async fn single_key_operations_round_trip_with_ttl_rounding( + #[with("single")] cache: Option, +) { + let Some(cache) = cache else { return }; let context = ExactCacheContext { ttl: Some(Duration::from_millis(1500)), }; let keys = multi_slot_keys(12); for (index, key) in keys.iter().enumerate() { cache - .set_cache(key, serde_json::json!({ "index": index }), &context) + .set_cache(key, json!({ "index": index }), &context) .unwrap(); } for (index, key) in keys.iter().enumerate() { assert_eq!( cache.get_cache(key, &context).unwrap(), - Some(serde_json::json!({ "index": index })) + Some(json!({ "index": index })) ); } - let runtime = tokio::runtime::Runtime::new().unwrap(); - let ttl = runtime.block_on(cache.async_get_ttl(&keys[0])).unwrap(); - assert_eq!(ttl, Some(2)); + assert_eq!(cache.async_get_ttl(&keys[0]).await.unwrap(), seconds(2)); + assert!( + cache + .async_refresh_ttl(&keys[0], seconds(40)) + .await + .unwrap() + ); + assert_eq!(cache.async_get_ttl(&keys[0]).await.unwrap(), seconds(40)); cache.delete_cache(&keys[0]).unwrap(); assert_eq!(cache.get_cache(&keys[0], &context).unwrap(), None); + assert!(!cache.async_refresh_ttl(&keys[0], None).await.unwrap()); assert!(cache.sync_ping().unwrap()); + cache.async_flush_cache().await.unwrap(); } +#[rstest] #[tokio::test] -async fn batch_reads_span_slots_and_preserve_order_with_malformed_entries() { - let cache = cluster_or_skip!("batch"); - let context = ExactCacheContext::default(); +async fn batch_reads_span_slots_and_preserve_order_with_malformed_entries( + #[with("batch")] cache: Option, + context: ExactCacheContext, +) { + let Some(cache) = cache else { return }; let keys = multi_slot_keys(40); for (index, key) in keys.iter().enumerate() { if index % 5 == 0 { continue; } cache - .async_set_cache(key, serde_json::json!(index), context.clone()) + .async_set_cache(key, json!(index), context.clone()) .await .unwrap(); } @@ -181,41 +144,65 @@ async fn batch_reads_span_slots_and_preserve_order_with_malformed_entries() { } else if index % 5 == 0 { BatchEntry::Miss } else { - BatchEntry::Hit(serde_json::json!(index)) + BatchEntry::Hit(json!(index)) }; assert_eq!(*entry, expected, "entry {index}"); } - let sync_entries = cache.batch_get_cache(&keys, &context).unwrap(); - assert_eq!(sync_entries, entries); + assert_eq!(cache.batch_get_cache(&keys, &context).unwrap(), entries); cache.delete_cache_keys(keys.clone()).await.unwrap(); let entries = cache.async_batch_get_cache(keys, context).await.unwrap(); assert!(entries.iter().all(|entry| *entry == BatchEntry::Miss)); } +#[rstest] #[tokio::test] -async fn pipelines_group_by_slot_and_return_results_in_submission_order() { - let cache = cluster_or_skip!("pipeline"); +async fn pipelines_group_by_slot_and_return_results_in_submission_order( + #[with("pipeline")] cache: Option, + counter: Option, + context: ExactCacheContext, +) { + let (Some(cache), Some(counter)) = (cache, counter) else { + return; + }; let keys = multi_slot_keys(30); - let entries = keys - .iter() - .enumerate() - .map(|(index, key)| (key.clone(), serde_json::json!(index))) - .collect(); cache - .async_set_cache_pipeline(entries, ExactCacheContext::default()) + .async_set_cache_pipeline( + keys.iter() + .enumerate() + .map(|(index, key)| (key.clone(), json!(index))) + .collect(), + context.clone(), + ) .await .unwrap(); let hits = cache - .async_batch_get_cache(keys.clone(), ExactCacheContext::default()) + .async_batch_get_cache(keys.clone(), context.clone()) .await .unwrap(); assert!( hits.iter() .enumerate() - .all(|(index, entry)| *entry == BatchEntry::Hit(serde_json::json!(index))) + .all(|(index, entry)| *entry == BatchEntry::Hit(json!(index))) ); + cache + .async_set_cache_pipeline_with_ttls( + keys.iter() + .enumerate() + .map(|(index, key)| (key.clone(), json!(index), seconds(index as u64 + 10))) + .collect(), + ) + .await + .unwrap(); + for (index, key) in keys.iter().enumerate() { + assert_eq!( + cache.async_get_ttl(key).await.unwrap(), + seconds(index as u64 + 10), + "{key}" + ); + } + let queues: Vec = keys.iter().map(|key| format!("queue:{key}")).collect(); let pushed = cache .async_rpush_pipeline( @@ -265,9 +252,6 @@ async fn pipelines_group_by_slot_and_return_results_in_submission_order() { } let counters: Vec = keys.iter().map(|key| format!("counter:{key}")).collect(); - let Some(counter) = counter_cache("counter") else { - return; - }; let totals = counter .async_increment_pipeline( counters @@ -284,25 +268,63 @@ async fn pipelines_group_by_slot_and_return_results_in_submission_order() { .unwrap(); let expected: Vec = (0..keys.len()).map(|index| index as f64 + 0.5).collect(); assert_eq!(totals, expected); - assert_eq!(counter.async_get_ttl(&counters[0]).await.unwrap(), Some(30)); + assert_eq!( + counter.async_get_ttl(&counters[0]).await.unwrap(), + seconds(30) + ); assert_eq!(counter.async_get_ttl(&counters[1]).await.unwrap(), None); counter.async_flush_cache().await.unwrap(); cache.async_flush_cache().await.unwrap(); } +#[rstest] #[tokio::test] -async fn scan_and_scoped_flush_cover_every_primary() { - let cache = cluster_or_skip!("flush"); - let other = cluster_or_skip!("other"); - let context = ExactCacheContext::default(); +async fn rpush_and_trim_is_one_transaction_on_the_key_slot( + #[with("trim")] cache: Option, +) { + let Some(cache) = cache else { return }; + let values = |values: &[&str]| values.iter().map(|value| RedisArg::from(*value)).collect(); + assert_eq!( + cache + .async_rpush_and_trim("buf", values(&["a", "b"]), 3) + .await + .unwrap(), + 2 + ); + assert_eq!( + cache + .async_rpush_and_trim("buf", values(&["c", "d"]), 3) + .await + .unwrap(), + 4 + ); + assert_eq!( + cache.async_lpop("buf", Some(10)).await.unwrap(), + RedisLpopResult::Values(vec![b"b".to_vec(), b"c".to_vec(), b"d".to_vec()]) + ); + cache.async_flush_cache().await.unwrap(); +} + +#[rstest] +#[tokio::test] +async fn scan_and_scoped_flush_cover_every_primary( + #[with("flush")] cache: Option, + #[from(cache)] + #[with("other")] + other: Option, + context: ExactCacheContext, +) { + let (Some(cache), Some(other)) = (cache, other) else { + return; + }; let keys = multi_slot_keys(60); for key in &keys { cache - .async_set_cache(key, serde_json::json!(true), context.clone()) + .async_set_cache(key, json!(true), context.clone()) .await .unwrap(); other - .async_set_cache(key, serde_json::json!(true), context.clone()) + .async_set_cache(key, json!(true), context.clone()) .await .unwrap(); } @@ -325,7 +347,7 @@ async fn scan_and_scoped_flush_cover_every_primary() { let kept = other.async_batch_get_cache(keys, context).await.unwrap(); assert!( kept.iter() - .all(|entry| *entry == BatchEntry::Hit(serde_json::json!(true))) + .all(|entry| *entry == BatchEntry::Hit(json!(true))) ); other.async_flush_cache().await.unwrap(); } @@ -361,9 +383,10 @@ fn ping_calls_per_node(startup: &redis::Client) -> Vec<(String, u64)> { counts } +#[rstest] #[tokio::test] -async fn ping_reaches_every_node() { - let cache = cluster_or_skip!("ping"); +async fn ping_reaches_every_node(#[with("ping")] cache: Option) { + let Some(cache) = cache else { return }; let startup = redis::Client::open(cluster_url()).unwrap(); let before = ping_calls_per_node(&startup); assert!(before.len() >= 2, "{before:?}"); @@ -375,14 +398,63 @@ async fn ping_reaches_every_node() { assert!(cache.sync_ping().unwrap()); let result = cache.test_connection().await.unwrap(); assert_eq!(result.status, CacheConnectionStatus::Success); + assert_eq!(result.message, "Redis Cluster connection test successful"); } +#[rstest] #[tokio::test] -async fn counters_claims_scripts_and_sets_work_on_the_cluster() { - let Some(counter) = counter_cache("counter") else { +async fn disconnect_closes_idle_connections_and_reconnects_on_demand( + #[with("disconnect")] cache: Option, +) { + let Some(cache) = cache else { return }; + assert!(cache.ping().await.unwrap()); + cache.disconnect().await.unwrap(); + assert!(cache.ping().await.unwrap()); +} + +#[rstest] +#[case::keep_existing_ttl(false)] +#[case::refresh_ttl(true)] +#[tokio::test] +async fn increments_refresh_the_ttl_only_when_asked( + counter: Option, + #[case] refresh_ttl: bool, +) { + let Some(counter) = counter else { return }; + let context = ExactCacheContext { ttl: seconds(60) }; + counter + .async_set_cache("spend", 0.0, ExactCacheContext { ttl: seconds(600) }) + .await + .unwrap(); + assert_eq!( + counter + .async_increment("spend", 1.5, context.clone(), refresh_ttl) + .await + .unwrap(), + 1.5 + ); + assert_eq!( + counter + .async_increment("spend", 2.0, context, refresh_ttl) + .await + .unwrap(), + 3.5 + ); + let ttl = counter.async_get_ttl("spend").await.unwrap().unwrap(); + assert_eq!(ttl <= Duration::from_secs(60), refresh_ttl, "{ttl:?}"); + counter.async_flush_cache().await.unwrap(); +} + +#[rstest] +#[tokio::test] +async fn counters_claims_scripts_and_sets_work_on_the_cluster( + counter: Option, + #[with("claim")] cache: Option, + context: ExactCacheContext, +) { + let (Some(counter), Some(cache)) = (counter, cache) else { return; }; - let context = ExactCacheContext::default(); assert_eq!( counter .increment_cache("spend", 1.5, context.clone()) @@ -391,7 +463,7 @@ async fn counters_claims_scripts_and_sets_work_on_the_cluster() { ); assert_eq!( counter - .async_increment("spend", 2.0, context.clone()) + .async_increment("spend", 2.0, context.clone(), false) .await .unwrap(), 3.5 @@ -413,9 +485,8 @@ async fn counters_claims_scripts_and_sets_work_on_the_cluster() { assert_eq!(counter.async_set_max("peak", 2.0, None).await.unwrap(), 4.0); counter.flush_cache().unwrap(); - let cache = cluster_or_skip!("claim"); - let owner = serde_json::json!("owner-a"); - let rival = serde_json::json!("owner-b"); + let owner = json!("owner-a"); + let rival = json!("owner-b"); assert_eq!( cache .claim_cache("lock", owner.clone(), &[], context.clone()) @@ -453,8 +524,8 @@ async fn counters_claims_scripts_and_sets_work_on_the_cluster() { .await .unwrap(); assert_eq!(reply, redis::Value::Okay); - assert_eq!(cache.async_get_ttl("scripted").await.unwrap(), Some(5)); - let evaluated: redis::Value = cache + assert_eq!(cache.async_get_ttl("scripted").await.unwrap(), seconds(5)); + let evaluated = cache .async_eval( "return redis.call('GET', KEYS[1])".into(), vec!["scripted".into()], @@ -472,13 +543,13 @@ async fn counters_claims_scripts_and_sets_work_on_the_cluster() { RedisArg::Bytes(b"a".to_vec()), RedisArg::Bytes(b"b".to_vec()) ], - Some(Duration::from_secs(9)), + seconds(9), ) .await .unwrap(), 2 ); - assert_eq!(cache.async_get_ttl("members").await.unwrap(), Some(9)); + assert_eq!(cache.async_get_ttl("members").await.unwrap(), seconds(9)); let result = cache.test_connection().await.unwrap(); assert_eq!(result.status, CacheConnectionStatus::Success); diff --git a/litellm-rust/crates/cache-redis/tests/contract.rs b/litellm-rust/crates/cache-redis/tests/contract.rs new file mode 100644 index 00000000000..85d043b90b2 --- /dev/null +++ b/litellm-rust/crates/cache-redis/tests/contract.rs @@ -0,0 +1,95 @@ +//! The shared cache contracts, run against the in-process fake connection and, when +//! `LITELLM_TEST_REDIS_CLUSTER_NODES` is set, against a live Redis Cluster. + +mod support; + +use std::time::Duration; + +use litellm_cache::{ExactCacheContext, JsonCodec}; +use litellm_cache_testing as contract; +use rstest::rstest; +use serde_json::json; +use support::{JsonCache, cluster_cache, fake_cache}; + +const PREFIX: &str = "contract:"; + +#[derive(Clone, Copy, Debug)] +enum Contract { + HitAndMiss, + SyncAsyncEquivalence, + OverwriteReplaces, + PipelineWritesEveryEntry, + BatchPreservesOrder, + DeleteRemovesKey, + FlushClears, + CounterAccumulates, +} + +#[derive(Clone, Copy, Debug)] +enum Server { + Fake, + Cluster, +} + +async fn check(contract: Contract, cache: &JsonCache) +where + C: redis::ConnectionLike + Send + 'static, +{ + let context = ExactCacheContext::default(); + match contract { + Contract::HitAndMiss => { + contract::hit_and_miss(cache, context, PREFIX, json!({"answer": 42})).await + } + Contract::SyncAsyncEquivalence => { + contract::sync_async_equivalence(cache, context, PREFIX, json!("first"), json!([2])) + .await + } + Contract::OverwriteReplaces => { + contract::overwrite_replaces(cache, context, PREFIX, json!(1), json!({"b": 2})).await + } + Contract::PipelineWritesEveryEntry => { + contract::pipeline_writes_every_entry( + cache, + context, + PREFIX, + vec![json!("a"), json!(2), json!({"c": true})], + ) + .await + } + Contract::BatchPreservesOrder => { + contract::batch_preserves_order(cache, context, PREFIX, json!("first"), json!(2)).await + } + Contract::DeleteRemovesKey => { + contract::delete_removes_key(cache, context, PREFIX, json!("value")).await + } + Contract::FlushClears => { + contract::flush_clears(cache, context, PREFIX, json!("value")).await + } + Contract::CounterAccumulates => contract::counter_accumulates(cache, context, PREFIX).await, + } +} + +#[rstest] +#[case::hit_and_miss(Contract::HitAndMiss)] +#[case::sync_async_equivalence(Contract::SyncAsyncEquivalence)] +#[case::overwrite_replaces(Contract::OverwriteReplaces)] +#[case::pipeline_writes_every_entry(Contract::PipelineWritesEveryEntry)] +#[case::batch_preserves_order(Contract::BatchPreservesOrder)] +#[case::delete_removes_key(Contract::DeleteRemovesKey)] +#[case::flush_clears(Contract::FlushClears)] +#[case::counter_accumulates(Contract::CounterAccumulates)] +#[tokio::test] +async fn redis_satisfies_the_cache_contract( + #[case] contract: Contract, + #[values(Server::Fake, Server::Cluster)] server: Server, +) { + match server { + Server::Fake => check(contract, &fake_cache("contract")).await, + Server::Cluster => { + let label = format!("{contract:?}"); + if let Some(cache) = cluster_cache(&label, Duration::from_secs(120), JsonCodec::new()) { + check(contract, &cache).await; + } + } + } +} diff --git a/litellm-rust/crates/cache-redis/tests/support/mod.rs b/litellm-rust/crates/cache-redis/tests/support/mod.rs new file mode 100644 index 00000000000..bff1e618582 --- /dev/null +++ b/litellm-rust/crates/cache-redis/tests/support/mod.rs @@ -0,0 +1,231 @@ +#![allow(dead_code)] + +use std::{ + collections::BTreeMap, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use litellm_cache::{CacheCodec, Error, JsonCodec}; +use litellm_cache_redis::{RedisCache, RedisNode, RedisTopology}; + +/// Encodes a byte behind a tag, so a value written with another tag decodes as invalid. +pub struct TaggedByteCodec(pub u8); + +impl CacheCodec for TaggedByteCodec { + type Value = u8; + + fn encode(&self, value: &u8) -> Result, Error> { + if *value > 127 { + return Err(Error::InvalidEntry); + } + Ok(vec![self.0, *value]) + } + + fn decode(&self, bytes: &[u8]) -> Result { + match bytes { + [tag, value] if *tag == self.0 => Ok(*value), + _ => Err(Error::InvalidEntry), + } + } +} + +/// A stateful in-process stand-in for a Redis server that understands the string commands the +/// shared contracts exercise, so they run without a live server. TTLs are accepted and ignored. +#[derive(Default)] +pub struct FakeRedis { + strings: BTreeMap, Vec>, +} + +impl FakeRedis { + fn run(&mut self, command: Vec>) -> redis::RedisResult { + let name = String::from_utf8_lossy(&command[0]).to_ascii_uppercase(); + let args = &command[1..]; + Ok(match name.as_str() { + "PING" => redis::Value::SimpleString("PONG".into()), + "SET" | "SETEX" => { + let value = if name == "SET" { &args[1] } else { &args[2] }; + self.strings.insert(args[0].clone(), value.clone()); + redis::Value::Okay + } + "GET" => self.get(&args[0]), + "MGET" => redis::Value::Array(args.iter().map(|key| self.get(key)).collect()), + "DEL" => { + let removed = args + .iter() + .filter(|key| self.strings.remove(*key).is_some()) + .count(); + redis::Value::Int(removed as i64) + } + "SCAN" => { + let pattern = &args[2]; + let keys = self + .strings + .keys() + .filter(|key| glob(pattern, key)) + .map(|key| redis::Value::BulkString(key.clone())) + .collect(); + redis::Value::Array(vec![ + redis::Value::BulkString(b"0".to_vec()), + redis::Value::Array(keys), + ]) + } + "EVAL" if args[0].windows(11).any(|window| window == b"INCRBYFLOAT") => { + self.increment_by_float(&args[2], &args[3]) + } + "INCRBYFLOAT" => self.increment_by_float(&args[0], &args[1]), + _ => { + return Err(redis::RedisError::from(( + redis::ErrorKind::Client, + "unsupported command", + name, + ))); + } + }) + } + + fn get(&self, key: &[u8]) -> redis::Value { + self.strings.get(key).map_or(redis::Value::Nil, |value| { + redis::Value::BulkString(value.clone()) + }) + } + + fn increment_by_float(&mut self, key: &[u8], amount: &[u8]) -> redis::Value { + let current = self + .strings + .get(key) + .map_or(0.0, |value| parse_float(value)); + let total = format!("{}", current + parse_float(amount)); + self.strings + .insert(key.to_vec(), total.clone().into_bytes()); + redis::Value::BulkString(total.into_bytes()) + } +} + +fn parse_float(bytes: &[u8]) -> f64 { + std::str::from_utf8(bytes).unwrap().parse().unwrap() +} + +/// Redis `MATCH` globbing for `*`, `?` and backslash escapes. +fn glob(pattern: &[u8], key: &[u8]) -> bool { + match pattern.split_first() { + None => key.is_empty(), + Some((b'*', rest)) => (0..=key.len()).any(|skip| glob(rest, &key[skip..])), + Some((b'?', rest)) => !key.is_empty() && glob(rest, &key[1..]), + Some((b'\\', [escaped, rest @ ..])) => { + key.first() == Some(escaped) && glob(rest, &key[1..]) + } + Some((literal, rest)) => key.first() == Some(literal) && glob(rest, &key[1..]), + } +} + +/// Splits RESP request bytes into the commands they carry. +fn commands(mut bytes: &[u8]) -> Vec>> { + fn line<'a>(bytes: &mut &'a [u8]) -> &'a [u8] { + let end = bytes + .windows(2) + .position(|window| window == b"\r\n") + .unwrap(); + let (line, rest) = bytes.split_at(end); + *bytes = &rest[2..]; + line + } + fn length(line: &[u8]) -> usize { + std::str::from_utf8(&line[1..]).unwrap().parse().unwrap() + } + let mut commands = Vec::new(); + while !bytes.is_empty() { + let count = length(line(&mut bytes)); + let command = (0..count) + .map(|_| { + let size = length(line(&mut bytes)); + let (argument, rest) = bytes.split_at(size); + bytes = &rest[2..]; + argument.to_vec() + }) + .collect(); + commands.push(command); + } + commands +} + +impl redis::ConnectionLike for FakeRedis { + fn req_packed_command(&mut self, cmd: &[u8]) -> redis::RedisResult { + let command = commands(cmd).into_iter().next().unwrap(); + self.run(command) + } + + fn req_packed_commands( + &mut self, + cmd: &[u8], + offset: usize, + count: usize, + ) -> redis::RedisResult> { + let replies = commands(cmd) + .into_iter() + .map(|command| self.run(command)) + .collect::>>()?; + Ok(replies.into_iter().skip(offset).take(count).collect()) + } + + fn get_db(&self) -> i64 { + 0 + } + + fn check_connection(&mut self) -> bool { + true + } + + fn is_open(&self) -> bool { + true + } +} + +pub type JsonCache = RedisCache, C>; + +pub fn fake_cache(namespace: &str) -> JsonCache { + RedisCache::with_connection(FakeRedis::default(), None, JsonCodec::new()) + .with_namespace(Some(namespace.into())) +} + +/// Startup nodes from `LITELLM_TEST_REDIS_CLUSTER_NODES` (`host:port,host:port`); tests that +/// need a live cluster skip when it is unset. +pub fn cluster_topology() -> Option { + let nodes = std::env::var("LITELLM_TEST_REDIS_CLUSTER_NODES").ok()?; + let startup_nodes = nodes + .split(',') + .map(|node| { + let (host, port) = node.trim().rsplit_once(':').expect("host:port"); + RedisNode { + host: host.to_string(), + port: port.parse().expect("port"), + } + }) + .collect(); + Some(RedisTopology::Cluster { startup_nodes }) +} + +pub fn cluster_url() -> String { + std::env::var("LITELLM_TEST_REDIS_CLUSTER_URL") + .unwrap_or_else(|_| "redis://127.0.0.1:7000".into()) +} + +pub fn unique_namespace(label: &str) -> String { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + format!("cluster-test:{label}:{nanos}") +} + +pub fn cluster_cache( + label: &str, + default_ttl: Duration, + codec: S, +) -> Option> { + let topology = cluster_topology()?; + Some( + RedisCache::connect(&cluster_url(), &topology, Some(default_ttl), codec) + .expect("cluster connection") + .with_namespace(Some(unique_namespace(label))), + ) +} diff --git a/litellm-rust/crates/cache-response/Cargo.toml b/litellm-rust/crates/cache-response/Cargo.toml index 04affb9872d..42a1afb2ba0 100644 --- a/litellm-rust/crates/cache-response/Cargo.toml +++ b/litellm-rust/crates/cache-response/Cargo.toml @@ -17,4 +17,5 @@ litellm-cache-memory.workspace = true litellm-cache-redis.workspace = true redis = "1.7.0" redis-test = "1.0.4" +rstest.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/cache-response/README.md b/litellm-rust/crates/cache-response/README.md index 46e561ddad1..dbad474c9e7 100644 --- a/litellm-rust/crates/cache-response/README.md +++ b/litellm-rust/crates/cache-response/README.md @@ -1,14 +1,16 @@ -# Response cache foundation +# Response cache `ResponseCache` adds request keys, independent read/write controls, response envelopes, and freshness checks to any `B: BaseCache` ## Ownership -`litellm-cache` defines typed storage and codec traits. Memory and Redis implement those traits without depending on response policy. Other consumers can store their own value types using the same backend implementations +`litellm-cache` defines typed storage, codec, and capability traits. `BaseCache` is only get, set, TTL, and pipeline writes. Everything else is an optional capability a backend implements only where its Python class defines the method: `DisconnectCache`, `ConnectionCache` (`test_connection`), `PingCache`, `BatchCache`, `DeleteCache`, `FlushCache`, counters, queues, TTL, scan, and scripts. Memory, Redis, disk, S3, GCS, and Azure Blob implement those traits without depending on response policy, so other consumers can store their own value types in the same backends + +Semantic backends (Redis, Valkey, Qdrant) are generic over their embedder and codec, and share one prompt and embedding contract from `litellm_cache::semantic`. They take a `SemanticCacheContext`, so `ResponseCache` drives them the same way it drives exact backends `litellm-cache-response` owns response keys, controls, entries, the Python-compatible response codec, and `WriteBuffer`, the backend-neutral deferred-write policy. It has no runtime dependency on a specific cache backend or Python -The Python bridge constructs backends and selects them through its private `NativeResponseCache` enum, which only dispatches. Generic Rust callers inject their backend directly. A native gateway can construct the same generic response service in its own host +`ExactResponseCache` is the object-safe view of a `ResponseCache` over an exact backend. `ConnectionProbe` is the object-safe `test_connection`, implemented only when the backend implements `ConnectionCache`, so a host holds one next to its `ExactResponseCache` and reports the operation as unsupported otherwise, as Python's `BaseCache` does. Lookup, store, batch, and flush never require it ## Native Rust use @@ -28,34 +30,22 @@ cache.store(&request, json!({"answer": 7}), now)?; assert_eq!(cache.async_lookup(&request, now).await?, Some(json!({"answer": 7}))); ``` -For Redis, inject `RedisCache::new(url, ttl, ResponseCacheCodec)` instead. Namespaces are optional and existing namespace prefixes are preserved. Sync operations check out independent connections from a bounded pool, while async callers, including counters and claims, move that blocking work off the executor. The pool skips the checkout PING and instead discards any connection whose command failed +For Redis, inject `RedisCache::new(url, ttl, ResponseCacheCodec)` instead. Namespaces are optional and existing namespace prefixes are preserved Callers supply Unix time for response freshness. Backend TTL uses its own clock. A read can reject an entry through `max_age` even while the backend still retains it -## Python integration boundary +## Python integration -The extension keeps a private test harness for memory and Redis single and batch response lookup and storage. Batch lookup returns ordered values plus missing indices for embedding partial-hit wiring. No bridge-only cache type is part of the public API +The bridge activates backends through the Rust catalog in `litellm/rust_bridge/catalog.py`. Every cache rule ships as `PYTHON_ONLY`, so SDK, Router, and proxy calls stay on Python and construct no native cache resources until a rule is changed + +When a rule selects a backend, the Python `Cache` facade builds the native runtime from its own configuration and routes its storage calls (sync and async lookup and store, and pipelined batch store) to it. Stream replay, embedding partial-hit merging, response reconstruction, and callbacks stay in Python on top of that native store. The Python backend object remains for its direct API Object responses are written as they are, and every other response shape is written as a serialized string, which is the pair of shapes Python reads. A string on the wire is therefore always a serialized response, so string-valued responses round trip. Typed backends such as memory never pass through the codec -The resolver reads the namespace's `cache` attribute each time it resolves. A captured binding retains the selected service for its operation, including background writes. `None` disables caching. Custom Python cache objects keep their original methods, arguments, returned awaitables, exceptions, and caller-task execution - -Python callbacks use the built-in `Cache` API, so a `Cache` subclass works unchanged. A batch lookup takes one original kwargs mapping per request and returns the list of `get_cache` or gathered `async_get_cache` results, while native bindings return `{values, missing_indices}`. A batch store hands the caller's original result to `async_add_cache_pipeline`. `ping` calls `ping`, and a flush goes to the facade's backend - -The private facade test harness checks object identity, method overrides, effective TTL, Redis namespace, memory capacity, and later configuration changes before selecting native execution. Its snapshot includes Redis connection settings, so a later `redis_kwargs` change, including an SSL option, selects Python callback execution. Buffered async writes honor `redis_flush_size`. Public activation must construct the shared native service from the initial Python Redis settings, including `litellm.default_redis_ttl` and SSL options. A buffered entry keeps the time it was produced, and a failed flush drops its batch instead of growing the buffer during an outage. The harness does not migrate entries or replace Python methods. Until activation configures one shared service, the Python facade and native test service can hold separate data. Existing public cache constructors remain on Python - -Native cache handles must be recreated after fork. The bridge releases the GIL around native operations, and Redis runs blocking connection operations off the async executor. Native errors propagate to the host, which owns the existing fail-open and logging policy - -The Redis backend also provides the primitives needed to preserve its direct Python surface later: TLS URLs, ping, bulk delete, counter batches, TTL, scan, set membership, raw queue push and pop, queue and counter pipelines, counter floor and maximum operations, script evaluation, client information, namespaced flush, and full flush. These are backend operations only and are not exported to Python by this PR. Memory provides TTL, oldest-key, and counter-pipeline operations +Native cache handles must be recreated after fork. Native errors propagate to the host, which owns the existing fail-open and logging policy ## Adding another backend -Implement `BaseCache` for the backend with its associated value type, and accept a `CacheCodec` when wire serialization is needed. `ResponseCache` then works without another response implementation. Add a concrete bridge enum variant and constructor only when exposing that backend to Python +Implement `BaseCache` for the backend with its associated value type and the capability traits its Python class supports, and accept a `CacheCodec` when wire serialization is needed. `ResponseCache` then works without another response implementation -Verify typed values, TTL precedence, missing entries, serialization failures, namespaces, batch ordering, and sync/async behavior. Run response fixtures with `ResponseCacheCodec`, including both Python envelope encodings, before enabling a public facade - -## Follow-up scope - -Public SDK, Router, and proxy activation still need constructor parity, stream replay, embedding partial-batch integration, response reconstruction, callback scheduling, and failure-policy integration. This foundation does not switch those request paths - -Redis cluster, disk, and cloud stores remain follow-ups. Semantic backends plug in through `SemanticCacheContext`, which carries the prompt inputs and metadata alongside the cache TTL. The generic dual cache takes read, write, and remote-failure policies, runs its async operations through the async L2 methods, and provides L2-first counters and atomic affinity claims. Errors propagate by default, and `RemoteFailurePolicy::UseLocal` opts key-value operations and claims into the local tier when L2 is unavailable. Claims compare decoded values, so a pin written by Python still matches. Public Router integration remains follow-up work. Reservations and pubsub still need explicit capabilities owned by their consuming features. Adding a cache backend does not establish those guarantees +Run the `litellm-cache-testing` contract checks the backend's capabilities allow, and run response fixtures with `ResponseCacheCodec`, including both Python envelope encodings, before adding a catalog rule diff --git a/litellm-rust/crates/cache-response/src/buffer.rs b/litellm-rust/crates/cache-response/src/buffer.rs index 606c21410c7..68f2348e28b 100644 --- a/litellm-rust/crates/cache-response/src/buffer.rs +++ b/litellm-rust/crates/cache-response/src/buffer.rs @@ -1,9 +1,9 @@ use std::{sync::Mutex, time::Duration}; -use litellm_cache::{BaseCache, Error, ExactCacheContext}; +use litellm_cache::Error; use serde_json::Value; -use crate::{CacheEntry, ResponseCache, ResponseCacheRequest}; +use crate::{ExactResponseCache, ResponseCacheRequest}; pub struct WriteBuffer { flush_size: usize, @@ -18,9 +18,9 @@ impl WriteBuffer { } } - pub async fn async_store>( + pub async fn async_store( &self, - cache: &ResponseCache, + cache: &dyn ExactResponseCache, request: &ResponseCacheRequest, response: Value, now: Duration, diff --git a/litellm-rust/crates/cache-response/src/exact.rs b/litellm-rust/crates/cache-response/src/exact.rs new file mode 100644 index 00000000000..16b79e4b11b --- /dev/null +++ b/litellm-rust/crates/cache-response/src/exact.rs @@ -0,0 +1,160 @@ +use std::{future::Future, pin::Pin, time::Duration}; + +use litellm_cache::{ + BaseCache, BatchCache, CacheConnectionResult, ConnectionCache, Error, ExactCacheContext, + FlushCache, +}; +use serde_json::Value; + +use crate::{CacheEntry, PartialHits, ResponseCache, ResponseCacheRequest}; + +type BoxFuture<'a, T> = Pin + Send + 'a>>; + +/// Object-safe view of a `ResponseCache` over an exact-match backend, so hosts can hold every +/// exact backend behind one pointer without erasing which backend it is elsewhere. +pub trait ExactResponseCache: Send + Sync { + fn default_ttl(&self) -> Option; + + fn lookup(&self, request: &ResponseCacheRequest, now: Duration) + -> Result, Error>; + + fn store( + &self, + request: &ResponseCacheRequest, + response: Value, + now: Duration, + ) -> Result<(), Error>; + + fn lookup_batch( + &self, + requests: &[ResponseCacheRequest], + now: Duration, + ) -> Result; + + fn async_lookup<'a>( + &'a self, + request: &'a ResponseCacheRequest, + now: Duration, + ) -> BoxFuture<'a, Result, Error>>; + + fn async_store<'a>( + &'a self, + request: &'a ResponseCacheRequest, + response: Value, + now: Duration, + ) -> BoxFuture<'a, Result<(), Error>>; + + fn async_lookup_batch<'a>( + &'a self, + requests: &'a [ResponseCacheRequest], + now: Duration, + ) -> BoxFuture<'a, Result>; + + fn async_store_batch<'a>( + &'a self, + entries: Vec<(ResponseCacheRequest, Value)>, + now: Duration, + ) -> BoxFuture<'a, Result<(), Error>>; + + fn async_store_entries<'a>( + &'a self, + entries: Vec<(ResponseCacheRequest, Value, Duration)>, + ) -> BoxFuture<'a, Result<(), Error>>; + + fn async_flush<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>>; +} + +/// Object-safe `test_connection` for the exact backends whose Python class defines it. Hosts hold +/// one next to their `ExactResponseCache` when the backend has it, and report the operation as +/// unsupported otherwise, as Python's `BaseCache.test_connection` does. +pub trait ConnectionProbe: Send + Sync { + fn test_connection<'a>(&'a self) -> BoxFuture<'a, Result>; +} + +impl ConnectionProbe for ResponseCache +where + B: ConnectionCache, + B::Context: Default + PartialEq, +{ + fn test_connection<'a>(&'a self) -> BoxFuture<'a, Result> { + Box::pin(ResponseCache::test_connection(self)) + } +} + +impl ExactResponseCache for ResponseCache +where + B: BaseCache + BatchCache + FlushCache, +{ + fn default_ttl(&self) -> Option { + ResponseCache::default_ttl(self) + } + + fn lookup( + &self, + request: &ResponseCacheRequest, + now: Duration, + ) -> Result, Error> { + ResponseCache::lookup(self, request, now) + } + + fn store( + &self, + request: &ResponseCacheRequest, + response: Value, + now: Duration, + ) -> Result<(), Error> { + ResponseCache::store(self, request, response, now) + } + + fn lookup_batch( + &self, + requests: &[ResponseCacheRequest], + now: Duration, + ) -> Result { + ResponseCache::lookup_batch(self, requests, now) + } + + fn async_lookup<'a>( + &'a self, + request: &'a ResponseCacheRequest, + now: Duration, + ) -> BoxFuture<'a, Result, Error>> { + Box::pin(ResponseCache::async_lookup(self, request, now)) + } + + fn async_store<'a>( + &'a self, + request: &'a ResponseCacheRequest, + response: Value, + now: Duration, + ) -> BoxFuture<'a, Result<(), Error>> { + Box::pin(ResponseCache::async_store(self, request, response, now)) + } + + fn async_lookup_batch<'a>( + &'a self, + requests: &'a [ResponseCacheRequest], + now: Duration, + ) -> BoxFuture<'a, Result> { + Box::pin(ResponseCache::async_lookup_batch(self, requests, now)) + } + + fn async_store_batch<'a>( + &'a self, + entries: Vec<(ResponseCacheRequest, Value)>, + now: Duration, + ) -> BoxFuture<'a, Result<(), Error>> { + Box::pin(ResponseCache::async_store_batch(self, entries, now)) + } + + fn async_store_entries<'a>( + &'a self, + entries: Vec<(ResponseCacheRequest, Value, Duration)>, + ) -> BoxFuture<'a, Result<(), Error>> { + Box::pin(ResponseCache::async_store_entries(self, entries)) + } + + fn async_flush<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>> { + Box::pin(ResponseCache::async_flush(self)) + } +} diff --git a/litellm-rust/crates/cache-response/src/lib.rs b/litellm-rust/crates/cache-response/src/lib.rs index 91b36ebe24b..a6a4bb3eb64 100644 --- a/litellm-rust/crates/cache-response/src/lib.rs +++ b/litellm-rust/crates/cache-response/src/lib.rs @@ -2,6 +2,7 @@ mod buffer; mod caching; mod codec; mod embedding; +mod exact; mod response; pub use buffer::WriteBuffer; @@ -11,4 +12,5 @@ pub use caching::{ }; pub use codec::ResponseCacheCodec; pub use embedding::PartialHits; +pub use exact::{ConnectionProbe, ExactResponseCache}; pub use response::{ResponseCache, ResponseCacheRequest}; diff --git a/litellm-rust/crates/cache-response/src/response.rs b/litellm-rust/crates/cache-response/src/response.rs index 5088402f125..e761c7157db 100644 --- a/litellm-rust/crates/cache-response/src/response.rs +++ b/litellm-rust/crates/cache-response/src/response.rs @@ -1,7 +1,9 @@ use std::{sync::Arc, time::Duration}; use litellm_cache::{ - BaseCache, BatchCache, BatchEntry, CacheConnectionResult, CacheContext, Error, FlushCache, + BaseCache, BatchCache, BatchEntry, CacheConnectionResult, CacheContext, ConnectionCache, Error, + FlushCache, + semantic::{SemanticCache, SemanticLookup}, }; use serde_json::Value; @@ -78,7 +80,10 @@ where self.backend.async_flush_cache().await } - pub async fn test_connection(&self) -> Result { + pub async fn test_connection(&self) -> Result + where + B: ConnectionCache, + { self.backend.test_connection().await } @@ -121,6 +126,43 @@ where Ok(Self::fresh_or_miss(entry, now, request.max_age)) } + /// `lookup` plus the similarity the semantic backend reports. Freshness applies to the + /// value only: Python stamps the similarity before its max-age check. + pub fn lookup_semantic( + &self, + request: &ResponseCacheRequest, + now: Duration, + ) -> Result, Error> + where + B: SemanticCache, + { + if !request.controls.reads() { + return Ok(SemanticLookup::miss(None)); + } + let lookup = self + .backend + .get_cache_with_similarity(&cache_key(&request.key), &request.context); + Self::fresh_semantic(lookup, now, request.max_age) + } + + pub async fn async_lookup_semantic( + &self, + request: &ResponseCacheRequest, + now: Duration, + ) -> Result, Error> + where + B: SemanticCache, + { + if !request.controls.reads() { + return Ok(SemanticLookup::miss(None)); + } + let lookup = self + .backend + .async_get_cache_with_similarity(&cache_key(&request.key), &request.context) + .await; + Self::fresh_semantic(lookup, now, request.max_age) + } + pub fn lookup_batch( &self, requests: &[ResponseCacheRequest], @@ -290,6 +332,21 @@ where Ok(PartialHits::new(values)) } + fn fresh_semantic( + lookup: Result, Error>, + now: Duration, + max_age: Option, + ) -> Result, Error> { + match lookup { + Ok(lookup) => Ok(SemanticLookup { + value: Self::fresh_or_miss(lookup.value, now, max_age), + similarity: lookup.similarity, + }), + Err(Error::InvalidEntry) => Ok(SemanticLookup::miss(None)), + Err(error) => Err(error), + } + } + fn fresh_or_miss( entry: Option, now: Duration, diff --git a/litellm-rust/crates/cache-response/tests/caching.rs b/litellm-rust/crates/cache-response/tests/caching.rs index 0e8ce9b3b1d..93c2eb3d16d 100644 --- a/litellm-rust/crates/cache-response/tests/caching.rs +++ b/litellm-rust/crates/cache-response/tests/caching.rs @@ -1,90 +1,174 @@ use litellm_cache_response::{ CacheControls, CacheKeyContext, CacheKeyField, CacheKeyInput, cache_key, get_cache_key, + should_use_cache, }; +use rstest::rstest; use sha2::{Digest, Sha256}; -#[test] -fn keys_match_python_order_groups_files_presets_and_namespaces() { - let mut input = CacheKeyInput { - fields: vec![ - CacheKeyField { - name: "model".into(), - value: Some("deployment".into()), - api_parameter: true, - internal_parameter: false, - }, - CacheKeyField { - name: "file".into(), - value: None, - api_parameter: true, - internal_parameter: false, - }, - ], - namespace: Some("team".into()), - ..Default::default() - }; +fn field(name: &str, value: Option<&str>) -> CacheKeyField { + CacheKeyField { + name: name.into(), + value: value.map(str::to_owned), + api_parameter: true, + internal_parameter: false, + } +} + +fn hash(preimage: &[u8]) -> String { + format!("{:x}", Sha256::digest(preimage)) +} + +#[rstest] +#[case::caching_group_and_checksum( CacheKeyContext { model_group: Some("group".into()), caching_groups: vec![(vec!["group".into()], "['group']".into())], file_checksum: Some("checksum".into()), ..Default::default() - } - .apply(&mut input); - assert_eq!( - cache_key(&input), - format!( - "team:{:x}", - Sha256::digest(b"model: ['group']file: checksum") - ) - ); - input.preset = Some("preset".into()); + }, + Some("team"), + "team:", + b"model: ['group']file: checksum".as_slice(), +)] +#[case::model_group_outside_caching_groups( + CacheKeyContext { + model_group: Some("group".into()), + caching_groups: vec![(vec!["other".into()], "['other']".into())], + file_object_name: Some("object".into()), + ..Default::default() + }, + None, + "", + b"model: groupfile: object".as_slice(), +)] +#[case::metadata_file_name_before_parameters( + CacheKeyContext { + metadata_file_name: Some("metadata".into()), + parameters_file_name: Some("parameters".into()), + ..Default::default() + }, + Some(""), + "", + b"model: deploymentfile: metadata".as_slice(), +)] +#[case::parameters_file_name_last( + CacheKeyContext { + parameters_file_name: Some("parameters".into()), + ..Default::default() + }, + None, + "", + b"model: deploymentfile: parameters".as_slice(), +)] +#[case::no_context_keeps_the_request_model( + CacheKeyContext::default(), + Some("team"), + "team:", + b"model: deployment".as_slice(), +)] +fn keys_match_python_order_groups_files_and_namespaces( + #[case] context: CacheKeyContext, + #[case] namespace: Option<&str>, + #[case] prefix: &str, + #[case] preimage: &[u8], +) { + let mut input = CacheKeyInput { + fields: vec![field("model", Some("deployment")), field("file", None)], + namespace: namespace.map(str::to_owned), + ..Default::default() + }; + context.apply(&mut input); + let expected = format!("{prefix}{}", hash(preimage)); + assert_eq!(cache_key(&input), expected); + assert_eq!(get_cache_key(&input), expected); +} + +#[rstest] +#[case::api_parameter(true, false, false, true)] +#[case::provider_parameter_when_included(false, false, true, true)] +#[case::provider_parameter_when_excluded(false, false, false, false)] +#[case::internal_parameter_never(false, true, true, false)] +fn keys_hash_api_and_opted_in_provider_parameters( + #[case] api_parameter: bool, + #[case] internal_parameter: bool, + #[case] include_provider_parameters: bool, + #[case] hashed: bool, +) { + let input = CacheKeyInput { + fields: vec![ + field("model", Some("a")), + CacheKeyField { + name: "extra".into(), + value: Some("x".into()), + api_parameter, + internal_parameter, + }, + ], + include_provider_parameters, + ..Default::default() + }; + let preimage: &[u8] = if hashed { + b"model: aextra: x" + } else { + b"model: a" + }; + assert_eq!(cache_key(&input), hash(preimage)); +} + +#[rstest] +#[case::without_namespace(None)] +#[case::with_namespace(Some("team"))] +fn preset_keys_are_used_verbatim(#[case] namespace: Option<&str>) { + let input = CacheKeyInput { + fields: vec![field("model", Some("a"))], + preset: Some("preset".into()), + namespace: namespace.map(str::to_owned), + ..Default::default() + }; + assert_eq!(cache_key(&input), "preset"); assert_eq!(get_cache_key(&input), "preset"); } -#[test] -fn cache_controls_honor_default_modes_and_directives() { - let enabled = CacheControls { - supported_call_type: true, - configured: true, - default_on: true, - ..Default::default() - }; - assert!(enabled.reads()); - assert!(enabled.writes()); - assert!( - !CacheControls { - default_on: false, - ..enabled - } - .reads() - ); - assert!( - CacheControls { - default_on: false, - use_cache: true, - ..enabled - } - .reads() - ); - assert!( - !CacheControls { - no_cache: true, - ..enabled - } - .reads() - ); - assert!( - !CacheControls { - no_store: true, - ..enabled - } - .writes() - ); - assert!( - !CacheControls { - caching: Some(false), - ..enabled - } - .writes() - ); +const ENABLED: CacheControls = CacheControls { + supported_call_type: true, + configured: true, + native_backend: false, + default_on: true, + caching: None, + no_cache: false, + no_store: false, + use_cache: false, +}; + +#[rstest] +#[case::enabled(ENABLED, true, true)] +#[case::default_off(CacheControls { default_on: false, ..ENABLED }, false, false)] +#[case::default_off_with_use_cache( + CacheControls { default_on: false, use_cache: true, ..ENABLED }, + true, + true +)] +#[case::no_cache(CacheControls { no_cache: true, ..ENABLED }, false, true)] +#[case::no_store(CacheControls { no_store: true, ..ENABLED }, true, false)] +#[case::no_cache_and_no_store( + CacheControls { no_cache: true, no_store: true, ..ENABLED }, + false, + false +)] +#[case::caching_disabled(CacheControls { caching: Some(false), ..ENABLED }, false, false)] +#[case::caching_enabled(CacheControls { caching: Some(true), ..ENABLED }, true, true)] +#[case::unsupported_call_type( + CacheControls { supported_call_type: false, ..ENABLED }, + false, + false +)] +#[case::unconfigured(CacheControls { configured: false, ..ENABLED }, false, false)] +fn cache_controls_honor_default_modes_and_directives( + #[case] controls: CacheControls, + #[case] reads: bool, + #[case] writes: bool, +) { + assert_eq!(controls.reads(), reads); + assert_eq!(controls.writes(), writes); + assert_eq!(should_use_cache(controls), reads || writes); } diff --git a/litellm-rust/crates/cache-response/tests/codec.rs b/litellm-rust/crates/cache-response/tests/codec.rs new file mode 100644 index 00000000000..4fb5a5094cb --- /dev/null +++ b/litellm-rust/crates/cache-response/tests/codec.rs @@ -0,0 +1,110 @@ +use litellm_cache::{CacheCodec, Error}; +use litellm_cache_response::{CacheEntry, ResponseCacheCodec}; +use rstest::rstest; +use serde_json::{Value, json}; + +fn entry(response: Value) -> CacheEntry { + CacheEntry { + timestamp: Some(100.0), + response, + } +} + +#[rstest] +#[case::python_literal_object( + br#"{'timestamp': 100.0, 'response': {'text': 'hello \\ world', 'flag': True, 'empty': None, 'list': [1, 2.5]}}"#.as_slice(), + json!({"text": "hello \\ world", "flag": true, "empty": null, "list": [1, 2.5]}), +)] +#[case::python_sync_string_response( + br#"{'timestamp': 100.0, 'response': '{"ok": true, "text": "cached"}'}"#.as_slice(), + json!({"ok": true, "text": "cached"}), +)] +#[case::python_literal_string_response( + br#"{'timestamp': 100.0, 'response': "{'ok': True, 'items': (1, 2)}"}"#.as_slice(), + json!({"ok": true, "items": [1, 2]}), +)] +#[case::json_object_response( + br#"{"timestamp":100.0,"response":{"ok":true,"text":"cached"}}"#.as_slice(), + json!({"ok": true, "text": "cached"}), +)] +#[case::json_string_response( + br#"{"timestamp": 100.0, "response": "[1,2]"}"#.as_slice(), + json!([1, 2]), +)] +fn decode_reads_every_python_envelope(#[case] bytes: &[u8], #[case] response: Value) { + assert_eq!(ResponseCacheCodec.decode(bytes).unwrap(), entry(response)); +} + +#[rstest] +#[case::code_is_not_executed(b"__import__('os').system('false')".to_vec())] +#[case::non_numeric_timestamp(b"{'timestamp': 'invalid', 'response': {}}".to_vec())] +#[case::infinite_timestamp(b"{'timestamp': 1e9999, 'response': {}}".to_vec())] +#[case::missing_response(br#"{"timestamp": 100.0}"#.to_vec())] +#[case::unserialized_string_response(br#"{"timestamp": 100.0, "response": "not serialized"}"#.to_vec())] +#[case::non_utf8(vec![0xff, 0xfe])] +#[case::deep_nesting(format!("{}None{}", "[".repeat(1000), "]".repeat(1000)).into_bytes())] +fn decode_rejects_invalid_entries(#[case] bytes: Vec) { + assert_eq!( + ResponseCacheCodec.decode(&bytes).unwrap_err(), + Error::InvalidEntry + ); +} + +#[rstest] +#[case::nan(f64::NAN)] +#[case::infinity(f64::INFINITY)] +fn encode_rejects_non_finite_timestamps(#[case] timestamp: f64) { + assert_eq!( + ResponseCacheCodec + .encode(&CacheEntry { + timestamp: Some(timestamp), + response: json!({}), + }) + .unwrap_err(), + Error::InvalidEntry + ); +} + +#[rstest] +#[case::object(json!({"choices": [{"text": "cached"}]}), json!({"choices": [{"text": "cached"}]}))] +#[case::array(json!([1, 2]), json!("[1,2]"))] +#[case::number(json!(7), json!("7"))] +#[case::null(json!(null), json!("null"))] +#[case::string(json!("hello world"), json!("\"hello world\""))] +#[case::numeric_string(json!("123"), json!("\"123\""))] +#[case::null_string(json!("null"), json!("\"null\""))] +fn encode_writes_python_readable_envelopes_that_round_trip( + #[case] response: Value, + #[case] wire_response: Value, +) { + let wire = ResponseCacheCodec.encode(&entry(response.clone())).unwrap(); + assert_eq!( + serde_json::from_slice::(&wire).unwrap(), + json!({"timestamp": 100.0, "response": wire_response}) + ); + assert_eq!(ResponseCacheCodec.decode(&wire).unwrap(), entry(response)); +} + +#[rstest] +fn object_entries_preserve_the_existing_json_representation() { + let entry = CacheEntry { + timestamp: Some(123.0), + response: json!({"choices": [{"text": "cached"}]}), + }; + let bytes = ResponseCacheCodec.encode(&entry).unwrap(); + assert_eq!(bytes, serde_json::to_vec(&entry).unwrap()); + assert_eq!(ResponseCacheCodec.decode(&bytes).unwrap(), entry); +} + +#[rstest] +#[case::json(br#"{"choices": [{"text": "legacy"}]}"#.as_slice())] +#[case::python_literal(br#"{'choices': [{'text': 'legacy'}]}"#.as_slice())] +fn values_without_timestamps_decode_as_bare_responses(#[case] bytes: &[u8]) { + assert_eq!( + ResponseCacheCodec.decode(bytes).unwrap(), + CacheEntry { + timestamp: None, + response: json!({"choices": [{"text": "legacy"}]}), + } + ); +} diff --git a/litellm-rust/crates/cache-response/tests/connection.rs b/litellm-rust/crates/cache-response/tests/connection.rs new file mode 100644 index 00000000000..bc24cf56846 --- /dev/null +++ b/litellm-rust/crates/cache-response/tests/connection.rs @@ -0,0 +1,123 @@ +mod support; + +use std::{sync::Arc, time::Duration}; + +use litellm_cache::CacheConnectionStatus; +use litellm_cache_memory::InMemoryCache; +use litellm_cache_response::{ + CacheEntry, ConnectionProbe, ExactResponseCache, ResponseCache, ResponseCacheRequest, +}; +use redis_test::MockCmd; +use rstest::rstest; +use serde_json::json; +use support::{keyed, memory, redis, request}; + +#[rstest] +#[case::reachable( + Ok("PONG"), + CacheConnectionStatus::Success, + "Redis connection test successful", + false +)] +#[case::unexpected_reply( + Ok("NOPE"), + CacheConnectionStatus::Failed, + "Redis ping returned False", + false +)] +#[case::connection_refused( + Err(redis::RedisError::from((redis::ErrorKind::Io, "connection refused"))), + CacheConnectionStatus::Failed, + "Redis connection failed:", + true +)] +#[tokio::test] +async fn connection_backends_are_reachable_as_a_probe( + #[case] reply: redis::RedisResult<&'static str>, + #[case] status: CacheConnectionStatus, + #[case] message: &str, + #[case] has_error: bool, +) { + let probe: Arc = + Arc::new(redis(vec![MockCmd::new(redis::cmd("PING"), reply)], None)); + + let result = probe.test_connection().await.unwrap(); + assert_eq!(result.status, status); + assert!(result.message.starts_with(message), "{}", result.message); + assert_eq!(result.error.is_some(), has_error); +} + +#[rstest] +#[tokio::test] +async fn one_service_serves_both_the_exact_cache_and_its_probe(request: ResponseCacheRequest) { + let service = Arc::new(redis( + vec![ + MockCmd::new(redis::cmd("PING"), Ok("PONG")), + MockCmd::new( + redis::cmd("GET").arg("tenant:key"), + Ok(br#"{"timestamp":100.0,"response":{"ok":true}}"#.to_vec()), + ), + ], + Some("tenant"), + )); + let probe: Arc = service.clone(); + let exact: Arc = service; + + assert_eq!( + probe.test_connection().await.unwrap().status, + CacheConnectionStatus::Success + ); + assert_eq!( + exact + .async_lookup(&request, Duration::from_secs(100)) + .await + .unwrap(), + Some(json!({"ok": true})) + ); +} + +/// The in-memory backend has no `test_connection`, as in Python, and still serves every response +/// operation. +#[rstest] +#[tokio::test] +async fn backends_without_a_connection_test_serve_every_response_operation( + #[from(memory)] service: Arc>>, + request: ResponseCacheRequest, +) { + let cache: Arc = service; + let now = Duration::from_secs(100); + let other = keyed("tenant:other"); + let missing = keyed("tenant:missing"); + + assert_eq!(cache.default_ttl(), Some(Duration::from_secs(600))); + cache.store(&request, json!({"v": 1}), now).unwrap(); + assert_eq!(cache.lookup(&request, now).unwrap(), Some(json!({"v": 1}))); + cache + .async_store(&other, json!({"v": 2}), now) + .await + .unwrap(); + assert_eq!( + cache.async_lookup(&other, now).await.unwrap(), + Some(json!({"v": 2})) + ); + + let requests = [request.clone(), missing.clone(), other.clone()]; + let partial = cache.lookup_batch(&requests, now).unwrap(); + assert_eq!( + partial.values, + vec![Some(json!({"v": 1})), None, Some(json!({"v": 2}))] + ); + assert_eq!(partial.missing_indices, vec![1]); + + cache + .async_store_batch(vec![(missing.clone(), json!({"v": 3}))], now) + .await + .unwrap(); + let partial = cache.async_lookup_batch(&requests, now).await.unwrap(); + assert!(partial.missing_indices.is_empty()); + assert_eq!(partial.values[1], Some(json!({"v": 3}))); + + cache.async_flush().await.unwrap(); + let partial = cache.async_lookup_batch(&requests, now).await.unwrap(); + assert_eq!(partial.missing_indices, vec![0, 1, 2]); +} diff --git a/litellm-rust/crates/cache-response/tests/response.rs b/litellm-rust/crates/cache-response/tests/response.rs index dcfc0301148..ec5e16f1367 100644 --- a/litellm-rust/crates/cache-response/tests/response.rs +++ b/litellm-rust/crates/cache-response/tests/response.rs @@ -1,3 +1,5 @@ +mod support; + use std::{ sync::{ Arc, Mutex, @@ -7,32 +9,22 @@ use std::{ }; use litellm_cache::{ - BaseCache, CacheCodec, CacheConnectionResult, CacheConnectionStatus, Error, - SemanticCacheContext, + BaseCache, Error, SemanticCacheContext, + semantic::{SemanticCache, SemanticLookup}, }; use litellm_cache_memory::InMemoryCache; -use litellm_cache_redis::RedisCache; use litellm_cache_response::{ - CacheEntry, CacheKeyField, CacheKeyInput, ResponseCache, ResponseCacheCodec, - ResponseCacheRequest, WriteBuffer, + CacheControls, CacheEntry, CacheKeyField, CacheKeyInput, ResponseCache, ResponseCacheRequest, + WriteBuffer, cache_key, }; -use redis_test::{MockCmd, MockRedisConnection}; -use serde_json::json; +use redis_test::MockCmd; +use rstest::rstest; +use serde_json::{Value, json}; +use support::{keyed, memory, redis, request}; -fn memory() -> Arc>> { - Arc::new(ResponseCache::new(Arc::new(InMemoryCache::new( - Some(8), - Some(Duration::from_secs(600)), - )))) -} - -fn request() -> ResponseCacheRequest { - ResponseCacheRequest::new(CacheKeyInput { - preset: Some("tenant:key".into()), - ..Default::default() - }) -} +type Memory = Arc>>; +#[derive(Default)] struct SemanticBackend { entries: Mutex>, contexts: Mutex>, @@ -67,50 +59,133 @@ impl BaseCache for SemanticBackend { .find(|(entry_key, _)| entry_key == key) .map(|(_, entry)| entry.clone())) } - - async fn disconnect(&self) -> Result<(), Error> { - Ok(()) - } - - async fn test_connection(&self) -> Result { - Ok(CacheConnectionResult { - status: CacheConnectionStatus::Success, - message: "ok".into(), - error: None, - }) - } } -#[test] -fn semantic_context_reaches_backend_for_store_and_lookup() { - let backend = Arc::new(SemanticBackend { - entries: Mutex::new(Vec::new()), - contexts: Mutex::new(Vec::new()), - }); +#[rstest] +#[tokio::test] +async fn semantic_context_reaches_backend_for_store_and_lookup( + request: ResponseCacheRequest, + #[values(false, true)] asynchronous: bool, +) { + let backend = Arc::new(SemanticBackend::default()); let cache = ResponseCache::new(backend.clone()); let context = SemanticCacheContext { messages: Some(json!([{"role": "user", "content": "hello"}])), ..Default::default() }; - let request = request().with_context(context.clone()); + let request = request.with_context(context.clone()); let response = json!({"answer": 42}); + let now = Duration::from_secs(100); - cache - .store(&request, response.clone(), Duration::from_secs(100)) - .unwrap(); + let hit = if asynchronous { + cache + .async_store(&request, response.clone(), now) + .await + .unwrap(); + cache.async_lookup(&request, now).await.unwrap() + } else { + cache.store(&request, response.clone(), now).unwrap(); + cache.lookup(&request, now).unwrap() + }; - assert_eq!( - cache.lookup(&request, Duration::from_secs(100)).unwrap(), - Some(response) - ); + assert_eq!(hit, Some(response)); assert_eq!( backend.contexts.lock().unwrap().as_slice(), &[context.clone(), context] ); } +/// A semantic backend that answers every read with one fixed lookup. +struct ScoredBackend(Result, Error>); + +impl BaseCache for ScoredBackend { + type Value = CacheEntry; + type Context = SemanticCacheContext; + + fn get_ttl(&self, _: &Self::Context) -> Option { + None + } + + fn set_cache(&self, _: &str, _: Self::Value, _: &Self::Context) -> Result<(), Error> { + Ok(()) + } + + fn get_cache(&self, key: &str, context: &Self::Context) -> Result, Error> { + self.get_cache_with_similarity(key, context) + .map(|lookup| lookup.value) + } +} + +impl SemanticCache for ScoredBackend { + fn get_cache_with_similarity( + &self, + _: &str, + _: &Self::Context, + ) -> Result, Error> { + self.0.clone() + } + + async fn async_get_cache_with_similarity( + &self, + key: &str, + context: &Self::Context, + ) -> Result, Error> { + self.get_cache_with_similarity(key, context) + } +} + +fn scored(timestamp: f64, similarity: f64) -> Result, Error> { + Ok(SemanticLookup { + value: Some(CacheEntry { + timestamp: Some(timestamp), + response: json!({"answer": 42}), + }), + similarity: Some(similarity), + }) +} + +#[rstest] +#[case::fresh_hit(scored(95.0, 0.95), true, Ok(SemanticLookup { value: Some(json!({"answer": 42})), similarity: Some(0.95) }))] +#[case::stale_hit_keeps_the_similarity( + scored(50.0, 0.95), + true, + Ok(SemanticLookup::miss(Some(0.95))) +)] +#[case::miss_keeps_the_similarity( + Ok(SemanticLookup::miss(Some(0.4))), + true, + Ok(SemanticLookup::miss(Some(0.4))) +)] +#[case::no_search(Ok(SemanticLookup::miss(None)), true, Ok(SemanticLookup::miss(None)))] +#[case::disabled_reads_skip_the_backend(scored(95.0, 0.95), false, Ok(SemanticLookup::miss(None)))] +#[case::invalid_entry_is_a_miss(Err(Error::InvalidEntry), true, Ok(SemanticLookup::miss(None)))] +#[case::backend_errors_propagate(Err(Error::Unavailable), true, Err(Error::Unavailable))] #[tokio::test] -async fn sync_and_async_consumers_share_keys_ttls_and_freshness() { +async fn semantic_lookup_applies_freshness_to_the_value_only( + #[case] backend: Result, Error>, + #[case] reads: bool, + #[case] expected: Result, Error>, + #[values(false, true)] asynchronous: bool, + request: ResponseCacheRequest, +) { + let cache = ResponseCache::new(Arc::new(ScoredBackend(backend))); + let mut request = request.with_context(SemanticCacheContext::default()); + request.max_age = Some(Duration::from_secs(10)); + request.controls.no_cache = !reads; + let now = Duration::from_secs(100); + + let lookup = if asynchronous { + cache.async_lookup_semantic(&request, now).await + } else { + cache.lookup_semantic(&request, now) + }; + + assert_eq!(lookup, expected); +} + +#[rstest] +#[tokio::test] +async fn sync_and_async_consumers_share_keys_ttls_and_freshness(mut request: ResponseCacheRequest) { let clock = Arc::new(AtomicU64::new(100)); let backend = Arc::new(InMemoryCache::with_clock( Some(8), @@ -121,7 +196,6 @@ async fn sync_and_async_consumers_share_keys_ttls_and_freshness() { }, )); let cache = ResponseCache::new(backend.clone()); - let mut request = request(); request.context.ttl = Some(Duration::from_secs(10)); request.max_age = Some(Duration::from_secs(5)); cache @@ -172,80 +246,164 @@ async fn sync_and_async_consumers_share_keys_ttls_and_freshness() { ); } +#[derive(Clone, Copy, Debug)] +enum Directive { + Plain, + NoCache, + NoStore, + DefaultOff, + UseCache, + CachingOff, + Unsupported, +} + +impl Directive { + fn apply(self, controls: &mut CacheControls) { + match self { + Self::Plain => {} + Self::NoCache => controls.no_cache = true, + Self::NoStore => controls.no_store = true, + Self::DefaultOff => controls.default_on = false, + Self::UseCache => { + controls.default_on = false; + controls.use_cache = true; + } + Self::CachingOff => controls.caching = Some(false), + Self::Unsupported => controls.supported_call_type = false, + } + } +} + +#[rstest] +#[case::plain(Directive::Plain, Directive::Plain, true)] +#[case::no_store_skips_the_write(Directive::NoStore, Directive::Plain, false)] +#[case::no_store_keeps_reads(Directive::Plain, Directive::NoStore, true)] +#[case::no_cache_keeps_writes(Directive::NoCache, Directive::Plain, true)] +#[case::no_cache_skips_the_read(Directive::Plain, Directive::NoCache, false)] +#[case::default_off_skips_the_write(Directive::DefaultOff, Directive::Plain, false)] +#[case::default_off_skips_the_read(Directive::Plain, Directive::DefaultOff, false)] +#[case::use_cache_opts_in_under_default_off(Directive::UseCache, Directive::UseCache, true)] +#[case::caching_off_skips_the_write(Directive::CachingOff, Directive::Plain, false)] +#[case::caching_off_skips_the_read(Directive::Plain, Directive::CachingOff, false)] +#[case::unsupported_call_type_skips_the_write(Directive::Unsupported, Directive::Plain, false)] +#[case::unsupported_call_type_skips_the_read(Directive::Plain, Directive::Unsupported, false)] #[tokio::test] -async fn directives_skip_io_and_keep_reads_and_writes_independent() { - let cache = memory(); - let mut request = request(); +async fn directives_skip_io_and_keep_reads_and_writes_independent( + memory: Memory, + request: ResponseCacheRequest, + #[case] write: Directive, + #[case] read: Directive, + #[case] hit: bool, + #[values(false, true)] asynchronous: bool, +) { let now = Duration::from_secs(100); - request.controls.no_store = true; - cache - .async_store(&request, json!({"v": 1}), now) - .await - .unwrap(); - assert_eq!(cache.lookup(&request, now).unwrap(), None); - request.controls.no_store = false; - request.controls.no_cache = true; - cache.store(&request, json!({"v": 2}), now).unwrap(); - assert_eq!(cache.async_lookup(&request, now).await.unwrap(), None); - request.controls.no_cache = false; - assert_eq!(cache.lookup(&request, now).unwrap(), Some(json!({"v": 2}))); - request.controls.default_on = false; - cache.store(&request, json!({"v": 3}), now).unwrap(); - assert_eq!(cache.lookup(&request, now).unwrap(), None); - request.controls.use_cache = true; - assert_eq!(cache.lookup(&request, now).unwrap(), Some(json!({"v": 2}))); - request.controls.supported_call_type = false; - assert_eq!(cache.lookup(&request, now).unwrap(), None); -} + let mut writer = request.clone(); + write.apply(&mut writer.controls); + let mut reader = request; + read.apply(&mut reader.controls); -#[tokio::test] -async fn redis_consumer_reads_python_sync_and_async_envelopes_and_writes_compatible_json() { - let connection = MockRedisConnection::new([ - MockCmd::new( - redis::cmd("GET").arg("tenant:key"), - Ok(br#"{'timestamp': 100.0, 'response': '{"ok": true, "text": "cached"}'}"#.to_vec()), - ), - MockCmd::new( - redis::cmd("GET").arg("tenant:key"), - Ok(br#"{"timestamp":100.0,"response":{"ok":true,"text":"cached"}}"#.to_vec()), - ), - MockCmd::new( - redis::cmd("SETEX") - .arg("tenant:key") - .arg(600) - .arg(br#"{"timestamp":100.0,"response":{"ok":true,"text":"cached"}}"#.as_slice()), - Ok("OK"), - ), - ]) - .assert_all_commands_consumed(); - let backend = RedisCache::with_connection(connection, None, ResponseCacheCodec) - .with_namespace(Some("tenant".into())); - let cache = ResponseCache::new(Arc::new(backend)); - let request = request(); - let expected = json!({"ok": true, "text": "cached"}); - assert_eq!( - cache.lookup(&request, Duration::from_secs(101)).unwrap(), - Some(expected.clone()) - ); - assert_eq!( - cache - .async_lookup(&request, Duration::from_secs(101)) + if asynchronous { + memory + .async_store(&writer, json!({"v": 1}), now) .await - .unwrap(), - Some(expected.clone()) + .unwrap(); + } else { + memory.store(&writer, json!({"v": 1}), now).unwrap(); + } + let found = if asynchronous { + memory.async_lookup(&reader, now).await.unwrap() + } else { + memory.lookup(&reader, now).unwrap() + }; + + assert_eq!( + found, + hit.then(|| json!({"v": 1})), + "{write:?} then {read:?}" + ); +} + +#[rstest] +#[case::python_sync_literal( + br#"{'timestamp': 100.0, 'response': '{"ok": true, "text": "cached"}'}"#.as_slice() +)] +#[case::python_async_json(br#"{"timestamp":100.0,"response":{"ok":true,"text":"cached"}}"#.as_slice())] +#[tokio::test] +async fn redis_consumer_reads_python_sync_and_async_envelopes( + request: ResponseCacheRequest, + #[case] stored: &[u8], + #[values(false, true)] asynchronous: bool, +) { + let cache = redis( + vec![MockCmd::new( + redis::cmd("GET").arg("tenant:key"), + Ok(stored.to_vec()), + )], + Some("tenant"), + ); + let now = Duration::from_secs(101); + let found = if asynchronous { + cache.async_lookup(&request, now).await.unwrap() + } else { + cache.lookup(&request, now).unwrap() + }; + assert_eq!(found, Some(json!({"ok": true, "text": "cached"}))); +} + +#[rstest] +#[case::object( + json!({"ok": true, "text": "cached"}), + br#"{"timestamp":100.0,"response":{"ok":true,"text":"cached"}}"#.as_slice() +)] +#[case::array(json!([1, 2]), br#"{"timestamp":100.0,"response":"[1,2]"}"#.as_slice())] +#[tokio::test] +async fn redis_consumer_writes_python_compatible_json( + request: ResponseCacheRequest, + #[case] response: Value, + #[case] wire: &[u8], +) { + let cache = redis( + vec![MockCmd::new( + redis::cmd("SETEX").arg("tenant:key").arg(600).arg(wire), + Ok("OK"), + )], + Some("tenant"), ); cache - .async_store(&request, expected, Duration::from_secs(100)) + .async_store(&request, response, Duration::from_secs(100)) .await .unwrap(); } +#[rstest] #[tokio::test] -async fn captured_service_keeps_the_selected_backend_for_background_writes() { - let original = memory(); +async fn invalid_entries_are_misses_and_disabled_reads_do_not_touch_redis( + mut request: ResponseCacheRequest, +) { + let cache = redis( + vec![MockCmd::new( + redis::cmd("GET").arg("tenant:key"), + Ok(b"invalid".to_vec()), + )], + None, + ); + request.controls.no_cache = true; + assert_eq!(cache.lookup(&request, Duration::ZERO).unwrap(), None); + request.controls.no_cache = false; + assert_eq!( + cache.async_lookup(&request, Duration::ZERO).await.unwrap(), + None + ); +} + +#[rstest] +#[tokio::test] +async fn captured_service_keeps_the_selected_backend_for_background_writes( + #[from(memory)] original: Memory, + #[from(memory)] replacement: Memory, + request: ResponseCacheRequest, +) { let captured = original.clone(); - let replacement = memory(); - let request = request(); let writer = tokio::spawn({ let request = request.clone(); async move { @@ -271,9 +429,13 @@ async fn captured_service_keeps_the_selected_backend_for_background_writes() { ); } -#[test] -fn generated_keys_preserve_namespace_and_explicit_keys() { - let cache = memory(); +#[rstest] +#[case::with_namespace(Some("tenant"))] +#[case::without_namespace(None)] +fn generated_keys_preserve_namespace_and_explicit_keys( + memory: Memory, + #[case] namespace: Option<&str>, +) { let key = CacheKeyInput { fields: vec![CacheKeyField { name: "model".into(), @@ -281,201 +443,125 @@ fn generated_keys_preserve_namespace_and_explicit_keys() { api_parameter: true, internal_parameter: false, }], - namespace: Some("tenant".into()), + namespace: namespace.map(str::to_owned), ..Default::default() }; let generated = ResponseCacheRequest::new(key.clone()); - let explicit = ResponseCacheRequest::new(CacheKeyInput { - preset: Some(litellm_cache_response::cache_key(&key)), - ..Default::default() - }); - cache + let explicit = keyed(&cache_key(&key)); + memory .store(&generated, json!({"value": 7}), Duration::from_secs(100)) .unwrap(); assert_eq!( - cache.lookup(&explicit, Duration::from_secs(100)).unwrap(), + memory.lookup(&explicit, Duration::from_secs(100)).unwrap(), Some(json!({"value":7})) ); } -#[test] -fn response_codec_accepts_python_literals_without_executing_code() { - let bytes = br#"{'timestamp': 100.0, 'response': {'text': 'hello \\ world', 'flag': True, 'empty': None, 'list': [1, 2.5]}}"#; - let entry = ResponseCacheCodec.decode(bytes).unwrap(); - assert_eq!( - entry.response, - json!({"text": "hello \\ world", "flag": true, "empty": null, "list": [1, 2.5]}) - ); - for bytes in [ - b"__import__('os').system('false')".as_slice(), - b"{'timestamp': 'invalid', 'response': {}}", - b"{'timestamp': 1e9999, 'response': {}}", - ] { - assert_eq!( - ResponseCacheCodec.decode(bytes).unwrap_err(), - Error::InvalidEntry - ); - } - let deep = format!("{}None{}", "[".repeat(1000), "]".repeat(1000)); - assert_eq!( - ResponseCacheCodec.decode(deep.as_bytes()).unwrap_err(), - Error::InvalidEntry - ); - assert_eq!( - ResponseCacheCodec - .encode(&CacheEntry { - timestamp: Some(f64::NAN), - response: json!({}) - }) - .unwrap_err(), - Error::InvalidEntry - ); -} - -#[tokio::test] -async fn invalid_entries_are_misses_and_disabled_reads_do_not_touch_redis() { - let connection = MockRedisConnection::new([MockCmd::new( - redis::cmd("GET").arg("tenant:key"), - Ok(b"invalid".to_vec()), - )]) - .assert_all_commands_consumed(); - let backend = RedisCache::with_connection(connection, None, ResponseCacheCodec); - let cache = ResponseCache::new(Arc::new(backend)); - let mut request = request(); - request.controls.no_cache = true; - assert_eq!(cache.lookup(&request, Duration::ZERO).unwrap(), None); - request.controls.no_cache = false; - assert_eq!( - cache.async_lookup(&request, Duration::ZERO).await.unwrap(), - None - ); -} - -#[test] -fn string_responses_round_trip_through_typed_and_wire_backends() { - let cache = ResponseCache::new(Arc::new(InMemoryCache::default())); +#[rstest] +#[case::text(json!("hello world"))] +#[case::numeric_text(json!("123"))] +#[case::null_text(json!("null"))] +#[case::array(json!([1, 2]))] +fn non_object_responses_round_trip_through_a_typed_backend( + memory: Memory, + request: ResponseCacheRequest, + #[case] response: Value, +) { let now = Duration::from_secs(100); - for response in [json!("hello world"), json!("123"), json!("null")] { - cache.store(&request(), response.clone(), now).unwrap(); - assert_eq!( - cache.lookup(&request(), now).unwrap(), - Some(response.clone()) - ); - - let wire = ResponseCacheCodec - .encode(&CacheEntry { - timestamp: Some(100.0), - response: response.clone(), - }) - .unwrap(); - assert_eq!(ResponseCacheCodec.decode(&wire).unwrap().response, response); - } + memory.store(&request, response.clone(), now).unwrap(); + assert_eq!(memory.lookup(&request, now).unwrap(), Some(response)); } -#[test] -fn non_object_responses_are_written_as_python_readable_serialized_strings() { - let wire = ResponseCacheCodec - .encode(&CacheEntry { - timestamp: Some(100.0), - response: json!([1, 2]), - }) - .unwrap(); - assert_eq!( - serde_json::from_slice::(&wire).unwrap(), - json!({"timestamp": 100.0, "response": "[1,2]"}) - ); - assert_eq!( - ResponseCacheCodec.decode(&wire).unwrap().response, - json!([1, 2]) - ); - assert_eq!( - ResponseCacheCodec.decode(br#"{"timestamp": 100.0, "response": "not serialized"}"#), - Err(Error::InvalidEntry) - ); -} - -#[test] -fn response_entries_preserve_the_existing_json_representation() { - let codec = ResponseCacheCodec; - let entry = CacheEntry { - timestamp: Some(123.0), - response: json!({"choices": [{"text": "cached"}]}), - }; - let bytes = codec.encode(&entry).unwrap(); - assert_eq!(bytes, serde_json::to_vec(&entry).unwrap()); - assert_eq!(codec.decode(&bytes).unwrap(), entry); -} - -#[test] -fn response_codec_preserves_values_without_timestamps() { - let codec = ResponseCacheCodec; - let raw = json!({"choices": [{"text": "legacy"}]}); - let entry = codec.decode(&serde_json::to_vec(&raw).unwrap()).unwrap(); - assert_eq!(entry.timestamp, None); - assert_eq!(entry.response, raw); - +#[rstest] +fn entries_without_timestamps_are_always_fresh(request: ResponseCacheRequest) { let backend = Arc::new(InMemoryCache::default()); - BaseCache::set_cache(backend.as_ref(), "tenant:key", entry, &Default::default()).unwrap(); + BaseCache::set_cache( + backend.as_ref(), + "tenant:key", + CacheEntry { + timestamp: None, + response: json!({"choices": [{"text": "legacy"}]}), + }, + &Default::default(), + ) + .unwrap(); let cache = ResponseCache::new(backend); + let mut request = request; + request.max_age = Some(Duration::from_secs(1)); assert_eq!( - cache.lookup(&request(), Duration::from_secs(100)).unwrap(), + cache.lookup(&request, Duration::from_secs(100)).unwrap(), Some(json!({"choices": [{"text": "legacy"}]})) ); } +#[rstest] #[tokio::test] -async fn batch_lookup_reports_partial_hits_and_batch_store_populates_misses() { - let cache = memory(); - let requests = ["hit", "miss", "disabled"].map(|key| { - ResponseCacheRequest::new(CacheKeyInput { - preset: Some(key.into()), - ..Default::default() - }) - }); - cache - .store(&requests[0], json!({"value": 1}), Duration::from_secs(100)) +async fn batch_lookup_reports_partial_hits_and_batch_store_populates_misses( + memory: Memory, + #[values(false, true)] asynchronous: bool, +) { + let now = Duration::from_secs(100); + let mut requests = ["hit", "miss", "disabled"].map(keyed).to_vec(); + memory + .store(&requests[0], json!({"value": 1}), now) .unwrap(); - let mut requests = requests.to_vec(); requests[2].controls.caching = Some(false); - let partial = cache - .async_lookup_batch(&requests, Duration::from_secs(100)) - .await - .unwrap(); + let partial = if asynchronous { + memory.async_lookup_batch(&requests, now).await.unwrap() + } else { + memory.lookup_batch(&requests, now).unwrap() + }; assert_eq!(partial.values, vec![Some(json!({"value": 1})), None, None]); assert_eq!(partial.missing_indices, vec![1, 2]); - cache + memory .async_store_batch( vec![ (requests[1].clone(), json!({"value": 2})), (requests[2].clone(), json!({"value": 3})), ], - Duration::from_secs(100), + now, ) .await .unwrap(); assert_eq!( - cache - .lookup(&requests[1], Duration::from_secs(100)) - .unwrap(), + memory.lookup(&requests[1], now).unwrap(), Some(json!({"value": 2})) ); requests[2].controls.caching = None; - assert_eq!( - cache - .lookup(&requests[2], Duration::from_secs(100)) - .unwrap(), - None - ); + assert_eq!(memory.lookup(&requests[2], now).unwrap(), None); } +#[rstest] #[tokio::test] -async fn deferred_entries_keep_the_time_they_were_produced() { - let cache = ResponseCache::new(Arc::new(InMemoryCache::default())); - let mut request = request(); +async fn batch_lookup_with_no_readable_request_skips_the_backend( + #[values(false, true)] asynchronous: bool, +) { + let cache = redis(Vec::new(), None); + let mut request = keyed("key"); + request.controls.no_cache = true; + let requests = [request.clone(), request]; + let partial = if asynchronous { + cache + .async_lookup_batch(&requests, Duration::ZERO) + .await + .unwrap() + } else { + cache.lookup_batch(&requests, Duration::ZERO).unwrap() + }; + assert_eq!(partial.values, vec![None, None]); + assert_eq!(partial.missing_indices, vec![0, 1]); +} + +#[rstest] +#[tokio::test] +async fn deferred_entries_keep_the_time_they_were_produced( + memory: Memory, + mut request: ResponseCacheRequest, +) { request.max_age = Some(Duration::from_secs(10)); - cache + memory .async_store_entries(vec![( request.clone(), json!({"answer": 7}), @@ -485,27 +571,29 @@ async fn deferred_entries_keep_the_time_they_were_produced() { .unwrap(); assert_eq!( - cache.lookup(&request, Duration::from_secs(110)).unwrap(), + memory.lookup(&request, Duration::from_secs(110)).unwrap(), Some(json!({"answer": 7})) ); assert_eq!( - cache.lookup(&request, Duration::from_secs(111)).unwrap(), + memory.lookup(&request, Duration::from_secs(111)).unwrap(), None ); } +#[rstest] #[tokio::test] -async fn write_buffer_flushes_at_its_size_and_keeps_each_produced_time() { - let cache = ResponseCache::new(Arc::new(InMemoryCache::default())); +async fn write_buffer_flushes_at_its_size_and_keeps_each_produced_time( + memory: Memory, + request: ResponseCacheRequest, +) { let buffer = WriteBuffer::new(2); - let mut first = request(); + let mut first = request; first.max_age = Some(Duration::from_secs(10)); - let mut second = request(); - second.key.preset = Some("tenant:other".into()); + let second = keyed("tenant:other"); buffer .async_store( - &cache, + memory.as_ref(), &first, json!({"answer": 7}), Duration::from_secs(100), @@ -513,13 +601,13 @@ async fn write_buffer_flushes_at_its_size_and_keeps_each_produced_time() { .await .unwrap(); assert_eq!( - cache.lookup(&first, Duration::from_secs(100)).unwrap(), + memory.lookup(&first, Duration::from_secs(100)).unwrap(), None ); buffer .async_store( - &cache, + memory.as_ref(), &second, json!({"answer": 8}), Duration::from_secs(200), @@ -527,37 +615,36 @@ async fn write_buffer_flushes_at_its_size_and_keeps_each_produced_time() { .await .unwrap(); assert_eq!( - cache.lookup(&first, Duration::from_secs(110)).unwrap(), + memory.lookup(&first, Duration::from_secs(110)).unwrap(), Some(json!({"answer": 7})) ); assert_eq!( - cache.lookup(&first, Duration::from_secs(111)).unwrap(), + memory.lookup(&first, Duration::from_secs(111)).unwrap(), None ); assert_eq!( - cache.lookup(&second, Duration::from_secs(200)).unwrap(), + memory.lookup(&second, Duration::from_secs(200)).unwrap(), Some(json!({"answer": 8})) ); } +#[rstest] #[tokio::test] -async fn write_buffer_clear_drops_pending_entries() { - let cache = ResponseCache::new(Arc::new(InMemoryCache::default())); +async fn write_buffer_clear_drops_pending_entries(memory: Memory, request: ResponseCacheRequest) { let buffer = WriteBuffer::new(2); - let mut other = request(); - other.key.preset = Some("tenant:other".into()); + let other = keyed("tenant:other"); let now = Duration::from_secs(100); buffer - .async_store(&cache, &request(), json!({"answer": 7}), now) + .async_store(memory.as_ref(), &request, json!({"answer": 7}), now) .await .unwrap(); buffer.clear().unwrap(); buffer - .async_store(&cache, &other, json!({"answer": 8}), now) + .async_store(memory.as_ref(), &other, json!({"answer": 8}), now) .await .unwrap(); - assert_eq!(cache.lookup(&request(), now).unwrap(), None); - assert_eq!(cache.lookup(&other, now).unwrap(), None); + assert_eq!(memory.lookup(&request, now).unwrap(), None); + assert_eq!(memory.lookup(&other, now).unwrap(), None); } diff --git a/litellm-rust/crates/cache-response/tests/support/mod.rs b/litellm-rust/crates/cache-response/tests/support/mod.rs new file mode 100644 index 00000000000..b992a937b70 --- /dev/null +++ b/litellm-rust/crates/cache-response/tests/support/mod.rs @@ -0,0 +1,40 @@ +use std::{sync::Arc, time::Duration}; + +use litellm_cache_memory::InMemoryCache; +use litellm_cache_redis::RedisCache; +use litellm_cache_response::{ + CacheEntry, CacheKeyInput, ResponseCache, ResponseCacheCodec, ResponseCacheRequest, +}; +use redis_test::{MockCmd, MockRedisConnection}; +use rstest::fixture; + +pub type MockedRedis = RedisCache; + +#[fixture] +pub fn memory() -> Arc>> { + Arc::new(ResponseCache::new(Arc::new(InMemoryCache::new( + Some(8), + Some(Duration::from_secs(600)), + )))) +} + +#[fixture] +pub fn request() -> ResponseCacheRequest { + keyed("tenant:key") +} + +pub fn keyed(key: &str) -> ResponseCacheRequest { + ResponseCacheRequest::new(CacheKeyInput { + preset: Some(key.into()), + ..Default::default() + }) +} + +/// A Redis response cache that must receive exactly `commands`, in order. +pub fn redis(commands: Vec, namespace: Option<&str>) -> ResponseCache { + let connection = MockRedisConnection::new(commands).assert_all_commands_consumed(); + ResponseCache::new(Arc::new( + RedisCache::with_connection(connection, None, ResponseCacheCodec) + .with_namespace(namespace.map(str::to_owned)), + )) +} diff --git a/litellm-rust/crates/cache-s3/Cargo.toml b/litellm-rust/crates/cache-s3/Cargo.toml index cdc17e732cb..c8150180e7c 100644 --- a/litellm-rust/crates/cache-s3/Cargo.toml +++ b/litellm-rust/crates/cache-s3/Cargo.toml @@ -10,11 +10,17 @@ litellm-cache.workspace = true litellm-auth-aws.workspace = true aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] } aws-credential-types = "1.3.0" -aws-smithy-types = "1.6.0" +aws-smithy-runtime-api = { version = "1.16.2", features = ["client", "http-1x"] } +aws-smithy-types = { version = "1.6.0", features = ["http-body-1-x"] } aws-types = "1.6.0" +futures-util.workspace = true +http.workspace = true +reqwest.workspace = true tokio.workspace = true [dev-dependencies] +litellm-cache-testing.workspace = true +rstest.workspace = true wiremock = "0.6.5" serde_json.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/litellm-rust/crates/cache-s3/src/auth.rs b/litellm-rust/crates/cache-s3/src/auth.rs index b7ca722cea3..fdf71fc011b 100644 --- a/litellm-rust/crates/cache-s3/src/auth.rs +++ b/litellm-rust/crates/cache-s3/src/auth.rs @@ -5,22 +5,22 @@ use aws_credential_types::{ use litellm_auth_aws::{AwsAuthConfig, resolve_credentials}; #[derive(Clone)] -pub(crate) struct Credentials { +pub struct S3Credentials { config: AwsAuthConfig, env: fn(&str) -> Option, } -impl Credentials { - pub(crate) fn new(config: AwsAuthConfig) -> Self { +impl S3Credentials { + pub fn new(config: AwsAuthConfig) -> Self { Self::with_env(config, |name| std::env::var(name).ok()) } - pub(crate) fn with_env(config: AwsAuthConfig, env: fn(&str) -> Option) -> Self { + pub fn with_env(config: AwsAuthConfig, env: fn(&str) -> Option) -> Self { Self { config, env } } } -impl ProvideCredentials for Credentials { +impl ProvideCredentials for S3Credentials { fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a> where Self: 'a, @@ -45,57 +45,8 @@ impl ProvideCredentials for Credentials { } } -impl std::fmt::Debug for Credentials { +impl std::fmt::Debug for S3Credentials { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Credentials").finish_non_exhaustive() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn explicit_keys_ignore_an_ambient_session_token() { - let provider = Credentials::with_env( - AwsAuthConfig { - access_key_id: Some("key".to_string()), - secret_access_key: Some("secret".to_string()), - region_name: Some("us-east-1".to_string()), - ..Default::default() - }, - |name| (name == "AWS_SESSION_TOKEN").then(|| "ambient".to_string()), - ); - let credentials = provider.provide_credentials().await.unwrap(); - assert_eq!(credentials.access_key_id(), "key"); - assert_eq!(credentials.secret_access_key(), "secret"); - assert_eq!(credentials.session_token(), None); - } - - #[tokio::test] - async fn explicit_keys_keep_their_session_token() { - let provider = Credentials::new(AwsAuthConfig { - access_key_id: Some("key".to_string()), - secret_access_key: Some("secret".to_string()), - session_token: Some("t".to_string()), - region_name: Some("us-east-1".to_string()), - ..Default::default() - }); - let credentials = provider.provide_credentials().await.unwrap(); - assert_eq!(credentials.session_token(), Some("t")); - } - - #[tokio::test] - async fn environment_keys_resolve_with_their_session_token() { - let provider = Credentials::with_env(AwsAuthConfig::default(), |name| match name { - "AWS_ACCESS_KEY_ID" => Some("env-key".to_string()), - "AWS_SECRET_ACCESS_KEY" => Some("env-secret".to_string()), - "AWS_SESSION_TOKEN" => Some("env-token".to_string()), - _ => None, - }); - let credentials = provider.provide_credentials().await.unwrap(); - assert_eq!(credentials.access_key_id(), "env-key"); - assert_eq!(credentials.secret_access_key(), "env-secret"); - assert_eq!(credentials.session_token(), Some("env-token")); + f.debug_struct("S3Credentials").finish_non_exhaustive() } } diff --git a/litellm-rust/crates/cache-s3/src/cache.rs b/litellm-rust/crates/cache-s3/src/cache.rs index 9c791f42c3b..91cd5e8ef54 100644 --- a/litellm-rust/crates/cache-s3/src/cache.rs +++ b/litellm-rust/crates/cache-s3/src/cache.rs @@ -10,13 +10,14 @@ use aws_sdk_s3::{ primitives::ByteStream, }; use aws_smithy_types::{DateTime, date_time::Format}; +use futures_util::future::try_join_all; use litellm_auth_aws::AwsAuthConfig; use litellm_cache::{ - BaseCache, BatchCache, CacheCodec, CacheConnectionResult, Error, ExactCacheContext, FlushCache, + BaseCache, BatchCache, CacheCodec, DisconnectCache, Error, ExactCacheContext, FlushCache, }; use tokio::runtime::Handle; -use crate::auth::Credentials; +use crate::{auth::S3Credentials, transport::ReqwestHttpClient}; pub struct S3Endpoint { pub url: String, @@ -41,12 +42,13 @@ pub struct S3Cache { } impl S3Cache { - pub fn new(config: S3CacheConfig, codec: C, runtime: Handle) -> Self { + pub fn new(config: S3CacheConfig, http: reqwest::Client, codec: C, runtime: Handle) -> Self { let endpoint_url: Option = config.endpoint.map(|endpoint| endpoint.url); let base = aws_sdk_s3::Config::builder() .behavior_version(BehaviorVersion::latest()) .region(Region::new(config.region.clone())) - .credentials_provider(Credentials::new(config.auth)) + .http_client(ReqwestHttpClient(http)) + .credentials_provider(S3Credentials::new(config.auth)) .request_checksum_calculation(RequestChecksumCalculation::WhenRequired) .response_checksum_validation(ResponseChecksumValidation::WhenRequired); let builder = match &endpoint_url { @@ -202,13 +204,26 @@ impl BaseCache for S3Cache { self.get(key).await } + async fn async_set_cache_pipeline( + &self, + entries: Vec<(String, Self::Value)>, + context: Self::Context, + ) -> Result<(), Error> { + let context = &context; + try_join_all( + entries + .into_iter() + .map(|(key, value)| async move { self.put(&key, value, context).await }), + ) + .await + .map(drop) + } +} + +impl DisconnectCache for S3Cache { async fn disconnect(&self) -> Result<(), Error> { Ok(()) } - - async fn test_connection(&self) -> Result { - Err(Error::UnsupportedOperation) - } } impl BatchCache for S3Cache {} diff --git a/litellm-rust/crates/cache-s3/src/lib.rs b/litellm-rust/crates/cache-s3/src/lib.rs index f6126dfa908..9be0210aad3 100644 --- a/litellm-rust/crates/cache-s3/src/lib.rs +++ b/litellm-rust/crates/cache-s3/src/lib.rs @@ -1,4 +1,6 @@ mod auth; mod cache; +mod transport; +pub use auth::S3Credentials; pub use cache::{S3Cache, S3CacheConfig, S3Endpoint}; diff --git a/litellm-rust/crates/cache-s3/src/transport.rs b/litellm-rust/crates/cache-s3/src/transport.rs new file mode 100644 index 00000000000..3e5ce578c31 --- /dev/null +++ b/litellm-rust/crates/cache-s3/src/transport.rs @@ -0,0 +1,49 @@ +use aws_smithy_runtime_api::client::{ + http::{ + HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector, + }, + orchestrator::HttpRequest, + result::ConnectorError, + runtime_components::RuntimeComponents, +}; +use aws_smithy_types::body::SdkBody; + +#[derive(Clone, Debug)] +pub(crate) struct ReqwestHttpClient(pub(crate) reqwest::Client); + +impl HttpClient for ReqwestHttpClient { + fn http_connector( + &self, + _: &HttpConnectorSettings, + _: &RuntimeComponents, + ) -> SharedHttpConnector { + SharedHttpConnector::new(self.clone()) + } +} + +impl HttpConnector for ReqwestHttpClient { + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + let client = self.0.clone(); + HttpConnectorFuture::new(async move { + let request = request + .try_into_http1x() + .map_err(|error| ConnectorError::other(error.into(), None))? + .map(reqwest::Body::wrap); + let request = reqwest::Request::try_from(request) + .map_err(|error| ConnectorError::other(error.into(), None))?; + let response = client.execute(request).await.map_err(|error| { + if error.is_timeout() { + ConnectorError::timeout(error.into()) + } else { + ConnectorError::io(error.into()) + } + })?; + let response = http::Response::from(response).map(SdkBody::from_body_1_x); + response + .try_into() + .map_err(|error: aws_smithy_runtime_api::http::HttpError| { + ConnectorError::other(error.into(), None) + }) + }) + } +} diff --git a/litellm-rust/crates/cache-s3/tests/auth.rs b/litellm-rust/crates/cache-s3/tests/auth.rs new file mode 100644 index 00000000000..59c5aa2e091 --- /dev/null +++ b/litellm-rust/crates/cache-s3/tests/auth.rs @@ -0,0 +1,57 @@ +use aws_credential_types::provider::ProvideCredentials; +use litellm_auth_aws::AwsAuthConfig; +use litellm_cache_s3::S3Credentials; +use rstest::rstest; + +fn explicit(session_token: Option<&str>) -> AwsAuthConfig { + AwsAuthConfig { + access_key_id: Some("key".to_string()), + secret_access_key: Some("secret".to_string()), + session_token: session_token.map(str::to_string), + region_name: Some("us-east-1".to_string()), + ..Default::default() + } +} + +fn ambient_token(name: &str) -> Option { + (name == "AWS_SESSION_TOKEN").then(|| "ambient".to_string()) +} + +fn environment_keys(name: &str) -> Option { + match name { + "AWS_ACCESS_KEY_ID" => Some("env-key".to_string()), + "AWS_SECRET_ACCESS_KEY" => Some("env-secret".to_string()), + "AWS_SESSION_TOKEN" => Some("env-token".to_string()), + _ => None, + } +} + +#[rstest] +#[case::explicit_keys_ignore_an_ambient_session_token( + explicit(None), ambient_token, ("key", "secret", None) +)] +#[case::explicit_keys_keep_their_session_token( + explicit(Some("t")), ambient_token, ("key", "secret", Some("t")) +)] +#[case::environment_keys_resolve_with_their_session_token( + AwsAuthConfig::default(), environment_keys, ("env-key", "env-secret", Some("env-token")) +)] +#[tokio::test] +async fn credentials_resolve( + #[case] config: AwsAuthConfig, + #[case] env: fn(&str) -> Option, + #[case] expected: (&str, &str, Option<&str>), +) { + let credentials = S3Credentials::with_env(config, env) + .provide_credentials() + .await + .unwrap(); + assert_eq!( + ( + credentials.access_key_id(), + credentials.secret_access_key(), + credentials.session_token(), + ), + expected + ); +} diff --git a/litellm-rust/crates/cache-s3/tests/cache.rs b/litellm-rust/crates/cache-s3/tests/cache.rs index 9a71656286b..53a51f8a289 100644 --- a/litellm-rust/crates/cache-s3/tests/cache.rs +++ b/litellm-rust/crates/cache-s3/tests/cache.rs @@ -1,41 +1,23 @@ +mod support; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use litellm_auth_aws::AwsAuthConfig; +use aws_smithy_types::{DateTime, date_time::Format}; use litellm_cache::{ - BaseCache, BatchCache, BatchEntry, Error, ExactCacheContext, FlushCache, JsonCodec, + BaseCache, BatchCache, BatchEntry, DisconnectCache, Error, ExactCacheContext, FlushCache, }; -use litellm_cache_s3::{S3Cache, S3CacheConfig, S3Endpoint}; +use litellm_cache_s3::S3CacheConfig; +use rstest::{fixture, rstest}; use serde_json::{Value, json}; -use tokio::runtime::Handle; +use support::FakeBucket; use wiremock::{ Mock, MockServer, ResponseTemplate, + http::HeaderMap, matchers::{method, path}, }; -fn config(endpoint: String) -> S3CacheConfig { - S3CacheConfig { - bucket: "cache-bucket".to_string(), - key_prefix: "team/".to_string(), - region: "us-east-1".to_string(), - endpoint: Some(S3Endpoint { url: endpoint }), - auth: AwsAuthConfig { - access_key_id: Some("key".to_string()), - secret_access_key: Some("secret".to_string()), - region_name: Some("us-east-1".to_string()), - ..Default::default() - }, - } -} - -fn cache(endpoint: &str) -> S3Cache> { - S3Cache::new( - config(endpoint.to_string()), - JsonCodec::::new(), - Handle::current(), - ) -} - -async fn mock_server() -> MockServer { +#[fixture] +async fn server() -> MockServer { let server = MockServer::start().await; Mock::given(method("PUT")) .respond_with(ResponseTemplate::new(200).insert_header("etag", "\"etag\"")) @@ -44,23 +26,25 @@ async fn mock_server() -> MockServer { server } -fn http_date_from(headers: &wiremock::http::HeaderMap, name: &str) -> Option { - use aws_smithy_types::{DateTime, date_time::Format}; +fn http_date_from(headers: &HeaderMap, name: &str) -> Option { headers .get(name) .and_then(|value| DateTime::from_str(value.to_str().ok()?, Format::HttpDate).ok()) .map(|date| UNIX_EPOCH + Duration::new(date.secs() as u64, date.subsec_nanos())) } +fn ttl(seconds: u64) -> ExactCacheContext { + ExactCacheContext { + ttl: Some(Duration::from_secs(seconds)), + } +} + +#[rstest] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn set_writes_python_metadata_with_and_without_ttl() { - let server = mock_server().await; - let cache = cache(&server.uri()); - let context = ExactCacheContext { - ttl: Some(Duration::from_secs(90)), - }; +async fn set_writes_python_metadata_with_and_without_ttl(#[future(awt)] server: MockServer) { + let cache = support::cache(&server.uri()); cache - .set_cache("alpha:beta", json!({"answer": 1}), &context) + .set_cache("alpha:beta", json!({"answer": 1}), &ttl(90)) .unwrap(); cache .set_cache("plain", json!({"answer": 2}), &ExactCacheContext::default()) @@ -110,61 +94,84 @@ async fn set_writes_python_metadata_with_and_without_ttl() { ); } +#[rstest] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn get_hit_miss_expired_and_invalid_entries() { - let server = mock_server().await; - Mock::given(method("GET")) - .and(path("/cache-bucket/team/hit")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({"answer": 3}))) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path("/cache-bucket/team/missing")) - .respond_with( - ResponseTemplate::new(404).set_body_string("NoSuchKey"), - ) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path("/cache-bucket/team/denied")) - .respond_with( - ResponseTemplate::new(403).set_body_string("AccessDenied"), - ) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path("/cache-bucket/team/expired")) - .respond_with( - ResponseTemplate::new(200) - .insert_header("expires", "Thu, 01 Jan 1970 00:00:00 GMT") - .set_body_json(json!({"answer": 4})), - ) - .mount(&server) - .await; - Mock::given(method("GET")) - .and(path("/cache-bucket/team/malformed")) - .respond_with(ResponseTemplate::new(200).set_body_string("not a cache entry")) - .mount(&server) - .await; - let cache = cache(&server.uri()); - let context = ExactCacheContext::default(); +async fn async_set_signs_with_the_configured_keys(#[future(awt)] server: MockServer) { + support::cache(&server.uri()) + .async_set_cache("key", json!({"answer": 1}), ttl(3600)) + .await + .unwrap(); - assert_eq!( - cache.get_cache("hit", &context).unwrap(), - Some(json!({"answer": 3})) + let requests = server.received_requests().await.unwrap(); + let request = &requests[0]; + assert_eq!(request.url.path(), "/cache-bucket/team/key"); + assert!( + request.headers["authorization"] + .to_str() + .unwrap() + .contains("Credential=key/") ); - assert_eq!(cache.get_cache("missing", &context).unwrap(), None); - assert_eq!(cache.get_cache("denied", &context).unwrap(), None); - assert_eq!(cache.get_cache("expired", &context).unwrap(), None); + assert!(request.headers.get("x-amz-security-token").is_none()); assert_eq!( - cache.get_cache("malformed", &context), - Err(Error::InvalidEntry) + request.headers["cache-control"].to_str().unwrap(), + "immutable, max-age=3600, s-maxage=3600" ); } +#[rstest] +#[case::hit("hit", ResponseTemplate::new(200).set_body_json(json!({"answer": 3})), Ok(Some(json!({"answer": 3}))))] +#[case::no_such_key( + "missing", + ResponseTemplate::new(404).set_body_string("NoSuchKey"), + Ok(None) +)] +#[case::access_denied( + "denied", + ResponseTemplate::new(403).set_body_string("AccessDenied"), + Ok(None) +)] +#[case::expired( + "expired", + ResponseTemplate::new(200) + .insert_header("expires", "Thu, 01 Jan 1970 00:00:00 GMT") + .set_body_json(json!({"answer": 4})), + Ok(None) +)] +#[case::not_yet_expired( + "fresh", + ResponseTemplate::new(200) + .insert_header("expires", "Fri, 01 Jan 2100 00:00:00 GMT") + .set_body_json(json!({"answer": 5})), + Ok(Some(json!({"answer": 5}))) +)] +#[case::malformed( + "malformed", + ResponseTemplate::new(200).set_body_string("not a cache entry"), + Err(Error::InvalidEntry) +)] +#[case::server_error("broken", ResponseTemplate::new(500), Err(Error::Unavailable))] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn batch_get_preserves_order_with_hits_misses_and_invalid() { - let server = mock_server().await; +async fn get_maps_s3_responses( + #[future(awt)] server: MockServer, + #[case] key: &str, + #[case] response: ResponseTemplate, + #[case] expected: Result, Error>, +) { + Mock::given(method("GET")) + .and(path(format!("/cache-bucket/team/{key}"))) + .respond_with(response) + .mount(&server) + .await; + let cache = support::cache(&server.uri()); + let context = ExactCacheContext::default(); + + assert_eq!(cache.get_cache(key, &context), expected); + assert_eq!(cache.async_get_cache(key, &context).await, expected); +} + +#[rstest] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn batch_get_preserves_order_with_hits_misses_and_invalid(#[future(awt)] server: MockServer) { for (key, status, body) in [ ("first", 200, "{\"answer\": 1}"), ("invalid", 200, "garbage"), @@ -180,91 +187,117 @@ async fn batch_get_preserves_order_with_hits_misses_and_invalid() { .respond_with(ResponseTemplate::new(404)) .mount(&server) .await; - let cache = cache(&server.uri()); + let cache = support::cache(&server.uri()); let context = ExactCacheContext::default(); let keys = vec![ "first".to_string(), "miss".to_string(), "invalid".to_string(), ]; + let expected = vec![ + BatchEntry::Hit(json!({"answer": 1})), + BatchEntry::Miss, + BatchEntry::Invalid, + ]; - let entries = cache.batch_get_cache(&keys, &context).unwrap(); - + assert_eq!(cache.batch_get_cache(&keys, &context).unwrap(), expected); assert_eq!( - entries, - vec![ - BatchEntry::Hit(json!({"answer": 1})), - BatchEntry::Miss, - BatchEntry::Invalid, - ] + cache.async_batch_get_cache(keys, context).await.unwrap(), + expected ); } +#[rstest] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn unsupported_and_noop_capabilities_match_python() { - let server = mock_server().await; - let cache = cache(&server.uri()); +async fn pipeline_writes_every_entry_with_the_shared_ttl() { + let server = FakeBucket::serve().await; + let cache = support::cache(&server.uri()); + cache + .async_set_cache_pipeline( + vec![ + ("one".into(), json!({"n": 1})), + ("two".into(), json!({"n": 2})), + ], + ttl(30), + ) + .await + .unwrap(); + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 2); + assert!(requests.iter().all(|request| { + request.headers["cache-control"].to_str().unwrap() == "immutable, max-age=30, s-maxage=30" + })); assert_eq!( - cache.test_connection().await, - Err(Error::UnsupportedOperation) + cache + .get_cache("two", &ExactCacheContext::default()) + .unwrap(), + Some(json!({"n": 2})) ); +} + +#[rstest] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn flush_and_disconnect_are_noops_like_python(#[future(awt)] server: MockServer) { + let cache = support::cache(&server.uri()); + cache.flush_cache().unwrap(); + cache.async_flush_cache().await.unwrap(); cache.disconnect().await.unwrap(); - assert_eq!(cache.get_ttl(&ExactCacheContext::default()), None); - assert_eq!( - cache.get_ttl(&ExactCacheContext { - ttl: Some(Duration::from_secs(45)), - }), - Some(Duration::from_secs(45)) - ); assert!(server.received_requests().await.unwrap().is_empty()); } -#[test] -fn key_conversion_prefixes_and_splits_colons() { +#[rstest] +#[case::without_ttl(ExactCacheContext::default(), None)] +#[case::with_ttl(ttl(45), Some(Duration::from_secs(45)))] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_ttl_reports_the_request_ttl( + #[case] context: ExactCacheContext, + #[case] expected: Option, +) { + assert_eq!( + support::cache("http://localhost").get_ttl(&context), + expected + ); +} + +#[rstest] +#[case::prefixed("team/", "a:b:c", "team/a/b/c")] +#[case::prefixed_plain("team/", "plain", "team/plain")] +#[case::unprefixed("", "a:b", "a/b")] +fn key_conversion_prefixes_and_splits_colons( + #[case] key_prefix: &str, + #[case] key: &str, + #[case] expected: &str, +) { let runtime = tokio::runtime::Builder::new_multi_thread() .worker_threads(1) .enable_all() .build() .unwrap(); - let _guard = runtime.enter(); - let cache = S3Cache::new( + let cache = support::cache_with( S3CacheConfig { - key_prefix: "team/".to_string(), - ..config("http://localhost".to_string()) + key_prefix: key_prefix.to_string(), + ..support::config("http://localhost") }, - JsonCodec::::new(), runtime.handle().clone(), ); assert_eq!(cache.bucket(), "cache-bucket"); - assert_eq!(cache.key_prefix(), "team/"); - assert_eq!(cache.to_s3_key("a:b:c"), "team/a/b/c"); - assert_eq!(cache.to_s3_key("plain"), "team/plain"); - - let unprefixed = S3Cache::new( - S3CacheConfig { - key_prefix: String::new(), - ..config("http://localhost".to_string()) - }, - JsonCodec::::new(), - runtime.handle().clone(), - ); - assert_eq!(unprefixed.to_s3_key("a:b"), "a/b"); + assert_eq!(cache.key_prefix(), key_prefix); + assert_eq!(cache.region(), "us-east-1"); + assert_eq!(cache.endpoint(), Some("http://localhost")); + assert_eq!(cache.to_s3_key(key), expected); } +#[rstest] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn sync_methods_block_inside_and_outside_the_runtime() { - let server = mock_server().await; - Mock::given(method("GET")) - .and(path("/cache-bucket/team/key")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({"answer": 9}))) - .mount(&server) - .await; +async fn sync_methods_block_outside_the_runtime() { + let server = FakeBucket::serve().await; let uri = server.uri(); - let cache = tokio::task::spawn_blocking(move || { - let cache = cache(&uri); + let handle = tokio::runtime::Handle::current(); + let cached = tokio::task::spawn_blocking(move || { + let cache = support::cache_with(support::config(&uri), handle); let context = ExactCacheContext::default(); cache .set_cache("key", json!({"answer": 9}), &context) @@ -274,5 +307,5 @@ async fn sync_methods_block_inside_and_outside_the_runtime() { .await .unwrap(); - assert_eq!(cache, Some(json!({"answer": 9}))); + assert_eq!(cached, Some(json!({"answer": 9}))); } diff --git a/litellm-rust/crates/cache-s3/tests/contract.rs b/litellm-rust/crates/cache-s3/tests/contract.rs new file mode 100644 index 00000000000..9855e2868ff --- /dev/null +++ b/litellm-rust/crates/cache-s3/tests/contract.rs @@ -0,0 +1,65 @@ +mod support; + +use litellm_cache::ExactCacheContext; +use litellm_cache_testing as contract; +use rstest::{fixture, rstest}; +use serde_json::json; +use support::{FakeBucket, JsonS3Cache}; +use wiremock::MockServer; + +struct S3 { + cache: JsonS3Cache, + _server: MockServer, +} + +#[fixture] +async fn s3() -> S3 { + let server = FakeBucket::serve().await; + S3 { + cache: support::cache(&server.uri()), + _server: server, + } +} + +#[fixture] +fn context() -> ExactCacheContext { + ExactCacheContext::default() +} + +const PREFIX: &str = "contract:"; + +#[rstest] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn hit_and_miss(#[future(awt)] s3: S3, context: ExactCacheContext) { + contract::hit_and_miss(&s3.cache, context, PREFIX, json!({"answer": 42})).await; +} + +#[rstest] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn sync_async_equivalence(#[future(awt)] s3: S3, context: ExactCacheContext) { + contract::sync_async_equivalence(&s3.cache, context, PREFIX, json!("first"), json!([2])).await; +} + +#[rstest] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn overwrite_replaces(#[future(awt)] s3: S3, context: ExactCacheContext) { + contract::overwrite_replaces(&s3.cache, context, PREFIX, json!(1), json!({"b": 2})).await; +} + +#[rstest] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn pipeline_writes_every_entry(#[future(awt)] s3: S3, context: ExactCacheContext) { + contract::pipeline_writes_every_entry( + &s3.cache, + context, + PREFIX, + vec![json!("a"), json!(2), json!({"c": true})], + ) + .await; +} + +#[rstest] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn batch_preserves_order(#[future(awt)] s3: S3, context: ExactCacheContext) { + contract::batch_preserves_order(&s3.cache, context, PREFIX, json!("first"), json!(2)).await; +} diff --git a/litellm-rust/crates/cache-s3/tests/support/mod.rs b/litellm-rust/crates/cache-s3/tests/support/mod.rs new file mode 100644 index 00000000000..046b042c66a --- /dev/null +++ b/litellm-rust/crates/cache-s3/tests/support/mod.rs @@ -0,0 +1,77 @@ +#![allow(dead_code)] + +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +use litellm_auth_aws::AwsAuthConfig; +use litellm_cache::JsonCodec; +use litellm_cache_s3::{S3Cache, S3CacheConfig, S3Endpoint}; +use serde_json::Value; +use tokio::runtime::Handle; +use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate, http::Method, matchers::any}; + +pub type JsonS3Cache = S3Cache>; + +pub fn config(endpoint: &str) -> S3CacheConfig { + S3CacheConfig { + bucket: "cache-bucket".to_string(), + key_prefix: "team/".to_string(), + region: "us-east-1".to_string(), + endpoint: Some(S3Endpoint { + url: endpoint.to_string(), + }), + auth: AwsAuthConfig { + access_key_id: Some("key".to_string()), + secret_access_key: Some("secret".to_string()), + region_name: Some("us-east-1".to_string()), + ..Default::default() + }, + } +} + +pub fn cache_with(config: S3CacheConfig, runtime: Handle) -> JsonS3Cache { + S3Cache::new(config, reqwest::Client::new(), JsonCodec::new(), runtime) +} + +pub fn cache(endpoint: &str) -> JsonS3Cache { + cache_with(config(endpoint), Handle::current()) +} + +/// An in-memory bucket: PUT stores the body under the request path, GET serves it or answers +/// `NoSuchKey`. +#[derive(Clone, Default)] +pub struct FakeBucket { + objects: Arc>>>, +} + +impl FakeBucket { + pub async fn serve() -> MockServer { + let server = MockServer::start().await; + Mock::given(any()) + .respond_with(Self::default()) + .mount(&server) + .await; + server + } +} + +impl Respond for FakeBucket { + fn respond(&self, request: &Request) -> ResponseTemplate { + let path = request.url.path().to_string(); + let mut objects = self.objects.lock().unwrap(); + match request.method { + Method::PUT => { + objects.insert(path, request.body.clone()); + ResponseTemplate::new(200).insert_header("etag", "\"etag\"") + } + Method::GET => match objects.get(&path) { + Some(body) => ResponseTemplate::new(200).set_body_bytes(body.clone()), + None => ResponseTemplate::new(404) + .set_body_string("NoSuchKey"), + }, + _ => ResponseTemplate::new(405), + } + } +} diff --git a/litellm-rust/crates/cache-testing/Cargo.toml b/litellm-rust/crates/cache-testing/Cargo.toml new file mode 100644 index 00000000000..674472df50d --- /dev/null +++ b/litellm-rust/crates/cache-testing/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "litellm-cache-testing" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[dependencies] +litellm-cache.workspace = true diff --git a/litellm-rust/crates/cache-testing/src/lib.rs b/litellm-rust/crates/cache-testing/src/lib.rs new file mode 100644 index 00000000000..c227c1e12dd --- /dev/null +++ b/litellm-rust/crates/cache-testing/src/lib.rs @@ -0,0 +1,211 @@ +//! Backend-neutral contract checks every cache backend runs from its own `rstest` suite. +//! +//! Each check takes the cache under test, the context to call it with, a key `prefix` that +//! keeps runs apart on shared servers, and distinct sample values. A check panics with the +//! violated invariant, so a backend test is one `#[rstest]` case per contract. + +use std::fmt::Debug; + +use litellm_cache::{BaseCache, BatchCache, BatchEntry, CounterCache, DeleteCache, FlushCache}; + +fn key(prefix: &str, name: &str) -> String { + format!("{prefix}{name}") +} + +/// A missing key reads as `None`, and a written key reads back through sync and async gets. +pub async fn hit_and_miss(cache: &B, context: B::Context, prefix: &str, value: B::Value) +where + B: BaseCache, + B::Value: Debug + PartialEq, +{ + let key = key(prefix, "hit-and-miss"); + assert_eq!( + cache.get_cache(&key, &context).unwrap(), + None, + "unwritten key must miss" + ); + assert_eq!( + cache.async_get_cache(&key, &context).await.unwrap(), + None, + "unwritten key must miss asynchronously" + ); + cache.set_cache(&key, value.clone(), &context).unwrap(); + assert_eq!( + cache.get_cache(&key, &context).unwrap(), + Some(value.clone()) + ); + assert_eq!( + cache.async_get_cache(&key, &context).await.unwrap(), + Some(value) + ); +} + +/// Sync and async writes land in the same store: each is visible to the other read path. +pub async fn sync_async_equivalence( + cache: &B, + context: B::Context, + prefix: &str, + first: B::Value, + second: B::Value, +) where + B: BaseCache, + B::Value: Debug + PartialEq, +{ + let async_written = key(prefix, "async-written"); + let sync_written = key(prefix, "sync-written"); + cache + .async_set_cache(&async_written, first.clone(), context.clone()) + .await + .unwrap(); + assert_eq!( + cache.get_cache(&async_written, &context).unwrap(), + Some(first) + ); + cache + .set_cache(&sync_written, second.clone(), &context) + .unwrap(); + assert_eq!( + cache + .async_get_cache(&sync_written, &context) + .await + .unwrap(), + Some(second) + ); +} + +/// A second write to a key replaces the first. +pub async fn overwrite_replaces( + cache: &B, + context: B::Context, + prefix: &str, + first: B::Value, + second: B::Value, +) where + B: BaseCache, + B::Value: Debug + PartialEq, +{ + let key = key(prefix, "overwrite"); + cache.set_cache(&key, first, &context).unwrap(); + cache.set_cache(&key, second.clone(), &context).unwrap(); + assert_eq!(cache.get_cache(&key, &context).unwrap(), Some(second)); +} + +/// `async_set_cache_pipeline` writes every entry, and an empty pipeline succeeds. +pub async fn pipeline_writes_every_entry( + cache: &B, + context: B::Context, + prefix: &str, + values: Vec, +) where + B: BaseCache, + B::Value: Debug + PartialEq, +{ + cache + .async_set_cache_pipeline(Vec::new(), context.clone()) + .await + .unwrap(); + let entries = values + .iter() + .enumerate() + .map(|(index, value)| (key(prefix, &format!("pipeline-{index}")), value.clone())) + .collect::>(); + cache + .async_set_cache_pipeline(entries.clone(), context.clone()) + .await + .unwrap(); + for (key, value) in entries { + assert_eq!( + cache.get_cache(&key, &context).unwrap(), + Some(value), + "{key}" + ); + } +} + +/// Batch reads answer in request order, with a `Miss` in place of each absent key. +pub async fn batch_preserves_order( + cache: &B, + context: B::Context, + prefix: &str, + first: B::Value, + second: B::Value, +) where + B: BatchCache, + B::Value: Debug + PartialEq, +{ + let keys = vec![ + key(prefix, "batch-first"), + key(prefix, "batch-missing"), + key(prefix, "batch-second"), + ]; + cache.set_cache(&keys[0], first.clone(), &context).unwrap(); + cache.set_cache(&keys[2], second.clone(), &context).unwrap(); + let expected = vec![ + BatchEntry::Hit(first), + BatchEntry::Miss, + BatchEntry::Hit(second), + ]; + assert_eq!(cache.batch_get_cache(&keys, &context).unwrap(), expected); + assert_eq!( + cache.async_batch_get_cache(keys, context).await.unwrap(), + expected + ); +} + +/// Sync and async deletes remove only the named key, and deleting a missing key succeeds. +pub async fn delete_removes_key(cache: &B, context: B::Context, prefix: &str, value: B::Value) +where + B: DeleteCache, + B::Value: Debug + PartialEq, +{ + let sync_deleted = key(prefix, "delete-sync"); + let async_deleted = key(prefix, "delete-async"); + let kept = key(prefix, "delete-kept"); + for key in [&sync_deleted, &async_deleted, &kept] { + cache.set_cache(key, value.clone(), &context).unwrap(); + } + cache.delete_cache(&sync_deleted).unwrap(); + cache.async_delete_cache(&async_deleted).await.unwrap(); + cache + .delete_cache(&key(prefix, "delete-never-written")) + .unwrap(); + assert_eq!(cache.get_cache(&sync_deleted, &context).unwrap(), None); + assert_eq!(cache.get_cache(&async_deleted, &context).unwrap(), None); + assert_eq!(cache.get_cache(&kept, &context).unwrap(), Some(value)); +} + +/// `flush_cache` and `async_flush_cache` each leave the cache empty. +pub async fn flush_clears(cache: &B, context: B::Context, prefix: &str, value: B::Value) +where + B: FlushCache, + B::Value: Debug + PartialEq, +{ + let key = key(prefix, "flush"); + cache.set_cache(&key, value.clone(), &context).unwrap(); + cache.flush_cache().unwrap(); + assert_eq!(cache.get_cache(&key, &context).unwrap(), None); + cache.set_cache(&key, value, &context).unwrap(); + cache.async_flush_cache().await.unwrap(); + assert_eq!(cache.get_cache(&key, &context).unwrap(), None); +} + +/// Sync and async increments accumulate on one counter, starting from zero. Whole-number +/// steps, since Python's disk cache restarts any counter whose stored value is not an `int`. +pub async fn counter_accumulates(cache: &B, context: B::Context, prefix: &str) +where + B: CounterCache, +{ + let key = key(prefix, "counter"); + assert_eq!( + cache.increment_cache(&key, 1.0, context.clone()).unwrap(), + 1.0 + ); + assert_eq!( + cache + .async_increment(&key, 2.0, context.clone(), false) + .await + .unwrap(), + 3.0 + ); + assert_eq!(cache.increment_cache(&key, -1.0, context).unwrap(), 2.0); +} diff --git a/litellm-rust/crates/cache-valkey-semantic/Cargo.toml b/litellm-rust/crates/cache-valkey-semantic/Cargo.toml index f98bb5a5fa8..da9bb60c853 100644 --- a/litellm-rust/crates/cache-valkey-semantic/Cargo.toml +++ b/litellm-rust/crates/cache-valkey-semantic/Cargo.toml @@ -8,13 +8,13 @@ repository.workspace = true [dependencies] litellm-cache.workspace = true litellm-cache-redis.workspace = true -litellm-cache-response.workspace = true redis = { version = "1.7.0", features = ["tls-rustls"] } -serde_json.workspace = true sha2.workspace = true -tokio.workspace = true uuid = { version = "1", features = ["v4"] } [dev-dependencies] +litellm-cache-testing.workspace = true redis-test = "1.0.4" rstest.workspace = true +serde_json.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/cache-valkey-semantic/src/cache.rs b/litellm-rust/crates/cache-valkey-semantic/src/cache.rs new file mode 100644 index 00000000000..947c0edc479 --- /dev/null +++ b/litellm-rust/crates/cache-valkey-semantic/src/cache.rs @@ -0,0 +1,245 @@ +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; + +use litellm_cache::{ + BaseCache, CacheCodec, Error, SemanticCacheContext, + semantic::{Embedder, SemanticCache, SemanticLookup, prompt_from_context}, +}; +use litellm_cache_redis::{RedisTopology, connection::Connections}; + +use crate::{ + ValkeySemanticConfig, + index::IndexState, + search::{embedding_bytes, scope_tag, search_document, write_document}, +}; + +/// `ValkeySemanticCache`: a semantic cache on valkey-search's TAG + VECTOR index. Values go +/// through the injected codec, so the response layer decides what a cached entry is. +pub struct ValkeySemanticCache { + connections: Arc>, + embedder: E, + codec: S, + config: ValkeySemanticConfig, + index_dimension: Arc>>, +} + +impl ValkeySemanticCache { + pub fn new( + url: &str, + embedder: E, + codec: S, + config: ValkeySemanticConfig, + ) -> Result { + Ok(Self { + connections: Arc::new(Connections::open(url, &RedisTopology::Standalone)?), + embedder, + codec, + config, + index_dimension: Arc::new(Mutex::new(None)), + }) + } +} + +impl ValkeySemanticCache +where + E: Embedder, + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + pub fn with_connection( + connection: C, + embedder: E, + codec: S, + config: ValkeySemanticConfig, + ) -> Self { + Self { + connections: Arc::new(Connections::fixed(connection)), + embedder, + codec, + config, + index_dimension: Arc::new(Mutex::new(None)), + } + } + + pub fn similarity_threshold(&self) -> f64 { + self.config.similarity_threshold + } + + pub fn index_name(&self) -> &str { + &self.config.index_name + } + + fn index_state(&self) -> IndexState { + IndexState { + name: self.config.index_name.clone(), + prefix: format!("{}:", self.config.index_name), + dimension: Arc::clone(&self.index_dimension), + similarity_threshold: self.config.similarity_threshold, + } + } + + fn decode(&self, lookup: SemanticLookup>) -> Result, Error> { + Ok(SemanticLookup { + value: lookup + .value + .map(|bytes| self.codec.decode(&bytes)) + .transpose()?, + similarity: lookup.similarity, + }) + } +} + +impl ValkeySemanticCache +where + E: Embedder, + S: CacheCodec + Clone, + C: redis::ConnectionLike + Send + 'static, +{ + /// The same index and connections behind a different embedder. + pub fn with_embedder(&self, embedder: E2) -> ValkeySemanticCache { + ValkeySemanticCache { + connections: Arc::clone(&self.connections), + embedder, + codec: self.codec.clone(), + config: self.config.clone(), + index_dimension: Arc::clone(&self.index_dimension), + } + } +} + +impl BaseCache for ValkeySemanticCache +where + E: Embedder, + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + type Value = S::Value; + type Context = SemanticCacheContext; + + fn get_ttl(&self, context: &Self::Context) -> Option { + context.ttl + } + + fn set_cache( + &self, + key: &str, + value: Self::Value, + context: &Self::Context, + ) -> Result<(), Error> { + let Some(prompt) = prompt_from_context(context) else { + return Ok(()); + }; + let embedding = self.embedder.embed(&prompt, context.metadata.as_ref())?; + let response = self.codec.encode(&value)?; + let index = self.index_state(); + self.connections.execute(|connection| { + write_document( + connection, + &index, + &scope_tag(key), + &prompt, + response, + embedding_bytes(&embedding), + self.get_ttl(context), + ) + }) + } + + fn get_cache(&self, key: &str, context: &Self::Context) -> Result, Error> { + self.get_cache_with_similarity(key, context) + .map(|lookup| lookup.value) + } + + async fn async_set_cache( + &self, + key: &str, + value: Self::Value, + context: Self::Context, + ) -> Result<(), Error> { + let Some(prompt) = prompt_from_context(&context) else { + return Ok(()); + }; + let embedding = self + .embedder + .async_embed(&prompt, context.metadata.as_ref()) + .await?; + let response = self.codec.encode(&value)?; + let index = self.index_state(); + let scope = scope_tag(key); + Connections::run_blocking(Arc::clone(&self.connections), move |connection| { + write_document( + connection, + &index, + &scope, + &prompt, + response, + embedding_bytes(&embedding), + context.ttl, + ) + }) + .await + } + + async fn async_get_cache( + &self, + key: &str, + context: &Self::Context, + ) -> Result, Error> { + self.async_get_cache_with_similarity(key, context) + .await + .map(|lookup| lookup.value) + } +} + +/// Python stamps a similarity of `0.0` when there is no prompt or no document in the key's +/// scope, and the closest document's similarity even when it misses the threshold. +impl SemanticCache for ValkeySemanticCache +where + E: Embedder, + S: CacheCodec, + C: redis::ConnectionLike + Send + 'static, +{ + fn get_cache_with_similarity( + &self, + key: &str, + context: &Self::Context, + ) -> Result, Error> { + let Some(prompt) = prompt_from_context(context) else { + return Ok(SemanticLookup::miss(Some(0.0))); + }; + let embedding = self.embedder.embed(&prompt, context.metadata.as_ref())?; + let index = self.index_state(); + let lookup = self.connections.execute(|connection| { + search_document( + connection, + &index, + &scope_tag(key), + embedding_bytes(&embedding), + ) + })?; + self.decode(lookup) + } + + async fn async_get_cache_with_similarity( + &self, + key: &str, + context: &Self::Context, + ) -> Result, Error> { + let Some(prompt) = prompt_from_context(context) else { + return Ok(SemanticLookup::miss(Some(0.0))); + }; + let embedding = self + .embedder + .async_embed(&prompt, context.metadata.as_ref()) + .await?; + let index = self.index_state(); + let scope = scope_tag(key); + let lookup = Connections::run_blocking(Arc::clone(&self.connections), move |connection| { + search_document(connection, &index, &scope, embedding_bytes(&embedding)) + }) + .await?; + self.decode(lookup) + } +} diff --git a/litellm-rust/crates/cache-valkey-semantic/src/config.rs b/litellm-rust/crates/cache-valkey-semantic/src/config.rs new file mode 100644 index 00000000000..c417652c27b --- /dev/null +++ b/litellm-rust/crates/cache-valkey-semantic/src/config.rs @@ -0,0 +1,8 @@ +/// `ValkeySemanticCache.DEFAULT_VALKEY_INDEX_NAME`. +pub const DEFAULT_INDEX_NAME: &str = "litellm_semantic_cache_index"; + +#[derive(Clone, Debug, PartialEq)] +pub struct ValkeySemanticConfig { + pub similarity_threshold: f64, + pub index_name: String, +} diff --git a/litellm-rust/crates/cache-valkey-semantic/src/index.rs b/litellm-rust/crates/cache-valkey-semantic/src/index.rs new file mode 100644 index 00000000000..a0b7be4ca7b --- /dev/null +++ b/litellm-rust/crates/cache-valkey-semantic/src/index.rs @@ -0,0 +1,100 @@ +use std::sync::{Arc, Mutex}; + +use litellm_cache::Error; +use litellm_cache_redis::connection::ConnectionRef; + +use crate::search::value_text; + +/// The valkey-search index one cache writes to, with the dimension it was last ensured for. +#[derive(Clone)] +pub(crate) struct IndexState { + pub(crate) name: String, + pub(crate) prefix: String, + pub(crate) dimension: Arc>>, + pub(crate) similarity_threshold: f64, +} + +/// `_ensure_index_sync` / `_ensure_index_async`: create the TAG + HNSW index once per dimension, +/// and accept an existing index unless it reports a different dimension. +pub(crate) fn ensure_index( + connection: &mut ConnectionRef<'_>, + index: &IndexState, + dimension: usize, +) -> Result<(), Error> { + if index + .dimension + .lock() + .map_err(|_| Error::Unavailable)? + .is_some_and(|existing| existing == dimension) + { + return Ok(()); + } + let create = redis::cmd("FT.CREATE") + .arg(&index.name) + .arg("ON") + .arg("HASH") + .arg("PREFIX") + .arg(1) + .arg(&index.prefix) + .arg("SCHEMA") + .arg("litellm_cache_key") + .arg("TAG") + .arg("embedding") + .arg("VECTOR") + .arg("HNSW") + .arg(6) + .arg("TYPE") + .arg("FLOAT32") + .arg("DIM") + .arg(dimension) + .arg("DISTANCE_METRIC") + .arg("COSINE") + .query::(connection) + .map(|_| ()) + .map_err(|error| error.to_string()); + if let Err(message) = create { + if !message.to_ascii_lowercase().contains("already exists") { + return Err(Error::Unavailable); + } + let info = redis::cmd("FT.INFO") + .arg(&index.name) + .query::(connection) + .map_err(|_| Error::Unavailable)?; + if index_dimension_from_info(&info).is_some_and(|existing| existing != dimension) { + return Err(Error::Unavailable); + } + } + *index.dimension.lock().map_err(|_| Error::Unavailable)? = Some(dimension); + Ok(()) +} + +/// `_extract_index_dim`: flatten each attribute one level and read the value after +/// `dimensions`. +fn index_dimension_from_info(value: &redis::Value) -> Option { + let redis::Value::Array(values) = value else { + return None; + }; + let attributes = values.windows(2).find_map(|pair| { + (value_text(&pair[0]).as_deref() == Some("attributes")).then_some(&pair[1]) + })?; + let redis::Value::Array(fields) = attributes else { + return None; + }; + fields.iter().find_map(|field| { + let redis::Value::Array(values) = field else { + return None; + }; + let values = values + .iter() + .flat_map(|value| match value { + redis::Value::Array(values) => values.as_slice(), + _ => std::slice::from_ref(value), + }) + .collect::>(); + values.windows(2).find_map(|pair| { + (value_text(pair[0]).as_deref() == Some("dimensions")) + .then(|| value_text(pair[1]).and_then(|value| value.parse().ok())) + .flatten() + }) + }) +} diff --git a/litellm-rust/crates/cache-valkey-semantic/src/lib.rs b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs index 6062ccc842c..f2f62bc95bd 100644 --- a/litellm-rust/crates/cache-valkey-semantic/src/lib.rs +++ b/litellm-rust/crates/cache-valkey-semantic/src/lib.rs @@ -1,1153 +1,7 @@ -use std::{ - future::Future, - sync::{Arc, Mutex}, - time::Duration, -}; +mod cache; +mod config; +mod index; +mod search; -use litellm_cache::{BaseCache, CacheCodec, CacheConnectionResult, Error, SemanticCacheContext}; -use litellm_cache_redis::{ - RedisTopology, - connection::{ConnectionRef, Connections}, -}; -use litellm_cache_response::CacheEntry; -use serde_json::Value; -use sha2::{Digest, Sha256}; -use uuid::Uuid; - -pub trait Embedder: Send + Sync + 'static { - fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error>; - - fn async_embed( - &self, - prompt: &str, - metadata: Option<&Value>, - ) -> impl Future, Error>> + Send; -} - -pub struct PreparedEmbedding(pub Vec); - -impl Embedder for PreparedEmbedding { - fn embed(&self, _prompt: &str, _metadata: Option<&Value>) -> Result, Error> { - Ok(self.0.clone()) - } - - async fn async_embed( - &self, - _prompt: &str, - _metadata: Option<&Value>, - ) -> Result, Error> { - Ok(self.0.clone()) - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct ValkeySemanticConfig { - pub similarity_threshold: f64, - pub index_name: String, -} - -pub const DEFAULT_INDEX_NAME: &str = "litellm_semantic_cache_index"; - -#[derive(Clone)] -struct IndexState { - name: String, - prefix: String, - dimension: Arc>>, - similarity_threshold: f64, -} - -pub struct ValkeySemanticCache< - E: Embedder, - S: CacheCodec, - C = redis::Connection, -> { - connections: Arc>, - embedder: E, - codec: S, - config: ValkeySemanticConfig, - index_dimension: Arc>>, -} - -impl ValkeySemanticCache -where - E: Embedder, - S: CacheCodec, -{ - pub fn new( - url: &str, - embedder: E, - codec: S, - config: ValkeySemanticConfig, - ) -> Result { - Ok(Self { - connections: Arc::new(Connections::open(url, &RedisTopology::Standalone)?), - embedder, - codec, - config, - index_dimension: Arc::new(Mutex::new(None)), - }) - } -} - -impl ValkeySemanticCache -where - E: Embedder, - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - pub fn with_connection( - connection: C, - embedder: E, - codec: S, - config: ValkeySemanticConfig, - ) -> Self { - Self { - connections: Arc::new(Connections::fixed(connection)), - embedder, - codec, - config, - index_dimension: Arc::new(Mutex::new(None)), - } - } - - pub fn similarity_threshold(&self) -> f64 { - self.config.similarity_threshold - } - - pub fn index_name(&self) -> &str { - &self.config.index_name - } - - fn index_state(&self) -> IndexState { - IndexState { - name: self.config.index_name.clone(), - prefix: format!("{}:", self.config.index_name), - dimension: Arc::clone(&self.index_dimension), - similarity_threshold: self.config.similarity_threshold, - } - } -} - -impl ValkeySemanticCache -where - E: Embedder, - S: CacheCodec + Clone, - C: redis::ConnectionLike + Send + 'static, -{ - pub fn with_embedder(&self, embedder: E2) -> ValkeySemanticCache { - ValkeySemanticCache { - connections: Arc::clone(&self.connections), - embedder, - codec: self.codec.clone(), - config: self.config.clone(), - index_dimension: Arc::clone(&self.index_dimension), - } - } -} - -impl BaseCache for ValkeySemanticCache -where - E: Embedder, - S: CacheCodec, - C: redis::ConnectionLike + Send + 'static, -{ - type Value = CacheEntry; - type Context = SemanticCacheContext; - - fn get_ttl(&self, context: &Self::Context) -> Option { - context.ttl - } - - fn set_cache( - &self, - key: &str, - value: Self::Value, - context: &Self::Context, - ) -> Result<(), Error> { - let Some(prompt) = prompt_from_context(context) else { - return Ok(()); - }; - let embedding = self.embedder.embed(&prompt, context.metadata.as_ref())?; - let scope = scope_tag(key); - let response = self.codec.encode(&value)?; - let vector = embedding_bytes(&embedding); - let index = self.index_state(); - self.connections.execute(|connection| { - write_document( - connection, - &index, - &scope, - &prompt, - response, - vector, - self.get_ttl(context), - ) - }) - } - - fn get_cache(&self, key: &str, context: &Self::Context) -> Result, Error> { - let Some(prompt) = prompt_from_context(context) else { - return Ok(None); - }; - let embedding = self.embedder.embed(&prompt, context.metadata.as_ref())?; - let scope = scope_tag(key); - let vector = embedding_bytes(&embedding); - let index = self.index_state(); - let response = self.connections.execute(|connection| { - search_document(connection, &index, &scope, vector, embedding.len()) - })?; - let Some(response) = response else { - return Ok(None); - }; - self.codec.decode(&response).map(Some) - } - - fn async_set_cache( - &self, - key: &str, - value: Self::Value, - context: Self::Context, - ) -> impl Future> + Send { - let key = key.to_owned(); - let prompt = prompt_from_context(&context); - let metadata = context.metadata.clone(); - async move { - let Some(prompt) = prompt else { - return Ok(()); - }; - let embedding = self - .embedder - .async_embed(&prompt, metadata.as_ref()) - .await?; - let connections = Arc::clone(&self.connections); - let index = self.index_state(); - let response = self.codec.encode(&value)?; - let vector = embedding_bytes(&embedding); - let scope = scope_tag(&key); - let ttl = context.ttl; - Connections::run_blocking(connections, move |connection| { - write_document(connection, &index, &scope, &prompt, response, vector, ttl) - }) - .await - } - } - - fn async_get_cache( - &self, - key: &str, - context: &Self::Context, - ) -> impl Future, Error>> + Send { - let key = key.to_owned(); - let prompt = prompt_from_context(context); - let metadata = context.metadata.clone(); - async move { - let Some(prompt) = prompt else { - return Ok(None); - }; - let embedding = self - .embedder - .async_embed(&prompt, metadata.as_ref()) - .await?; - let connections = Arc::clone(&self.connections); - let index = self.index_state(); - Connections::run_blocking(connections, move |connection| { - let scope = scope_tag(&key); - let vector = embedding_bytes(&embedding); - search_document(connection, &index, &scope, vector, embedding.len()) - }) - .await - .and_then(|response| response.map(|bytes| self.codec.decode(&bytes)).transpose()) - } - } - - async fn disconnect(&self) -> Result<(), Error> { - Ok(()) - } - - async fn test_connection(&self) -> Result { - Err(Error::UnsupportedOperation) - } -} - -pub fn prompt_from_context(context: &SemanticCacheContext) -> Option { - if let Some(Value::Array(messages)) = context.messages.as_ref() - && !messages.is_empty() - { - return messages - .iter() - .filter_map(Value::as_object) - .map(message_text) - .collect(); - } - let input = context.input.as_ref()?; - let mut parts = Vec::new(); - collect_input_text(input, &mut parts); - let prompt = parts.join("\n").trim().to_owned(); - (!prompt.is_empty()).then_some(prompt) -} - -fn message_text(message: &serde_json::Map) -> Option { - let content = match message.get("content") { - Some(Value::String(value)) => value.clone(), - Some(Value::Array(parts)) => { - let mut content = String::new(); - for part in parts { - let part = part.as_object()?; - if let Some(text) = part.get("text").and_then(Value::as_str) { - content.push_str(text); - } - } - content - } - _ => String::new(), - }; - Some(format!( - "{content}{}", - search_results_text(message.get("search_results")) - )) -} - -fn search_results_text(value: Option<&Value>) -> String { - let Some(Value::Array(results)) = value else { - return String::new(); - }; - results - .iter() - .filter_map(Value::as_object) - .map(|result| { - let source = result.get("source").and_then(Value::as_str).unwrap_or(""); - let title = result.get("title").and_then(Value::as_str).unwrap_or(""); - let content = result - .get("content") - .and_then(Value::as_array) - .map(|blocks| { - blocks - .iter() - .filter_map(Value::as_object) - .filter_map(|block| block.get("text").and_then(Value::as_str)) - .collect::() - }) - .unwrap_or_default(); - let citations = result - .get("citations") - .filter(|value| !value.is_null()) - .and_then(|value| serde_json::to_string(value).ok()) - .unwrap_or_default(); - format!("{source}{title}{content}{citations}") - }) - .collect() -} - -fn collect_input_text(value: &Value, parts: &mut Vec) { - match value { - Value::String(value) => { - let value = value.trim(); - if !value.is_empty() { - parts.push(value.to_owned()); - } - } - Value::Array(values) => values - .iter() - .for_each(|value| collect_input_text(value, parts)), - Value::Object(object) => { - if let Some(content) = object.get("content").filter(|value| !value.is_null()) { - collect_input_text(content, parts); - return; - } - for key in ["text", "output", "input_text", "output_text"] { - if let Some(Value::String(value)) = object.get(key) { - let value = value.trim(); - if !value.is_empty() { - parts.push(value.to_owned()); - return; - } - } - } - } - _ => {} - } -} - -fn scope_tag(key: &str) -> String { - let digest = Sha256::digest(key.as_bytes()); - digest.iter().map(|byte| format!("{byte:02x}")).collect() -} - -fn embedding_bytes(embedding: &[f32]) -> Vec { - embedding - .iter() - .flat_map(|value| value.to_le_bytes()) - .collect() -} - -fn write_document( - connection: &mut ConnectionRef<'_>, - index: &IndexState, - scope: &str, - prompt: &str, - response: Vec, - vector: Vec, - ttl: Option, -) -> Result<(), Error> { - let dimension = vector.len() / std::mem::size_of::(); - ensure_index( - connection, - &index.name, - &index.prefix, - &index.dimension, - dimension, - )?; - let document = format!("{}{scope}:{}", index.prefix, Uuid::new_v4()); - let mut pipeline = redis::pipe(); - pipeline - .cmd("HSET") - .arg(&document) - .arg("litellm_cache_key") - .arg(scope) - .arg("prompt") - .arg(prompt) - .arg("response") - .arg(response) - .arg("embedding") - .arg(vector) - .ignore(); - if let Some(ttl) = ttl { - pipeline - .cmd("EXPIRE") - .arg(&document) - .arg(ttl.as_secs()) - .ignore(); - } - pipeline - .query::<()>(connection) - .map_err(|_| Error::Unavailable) -} - -fn search_document( - connection: &mut ConnectionRef<'_>, - index: &IndexState, - scope: &str, - vector: Vec, - dimension: usize, -) -> Result>, Error> { - ensure_index( - connection, - &index.name, - &index.prefix, - &index.dimension, - dimension, - )?; - let query = - format!("(@litellm_cache_key:{{{scope}}})=>[KNN 1 @embedding $vec AS vector_distance]"); - let response = redis::cmd("FT.SEARCH") - .arg(&index.name) - .arg(query) - .arg("PARAMS") - .arg(2) - .arg("vec") - .arg(vector) - .arg("RETURN") - .arg(2) - .arg("response") - .arg("vector_distance") - .arg("DIALECT") - .arg(2) - .query::(connection) - .map_err(|_| Error::Unavailable)?; - let Some(fields) = search_fields(response)? else { - return Ok(None); - }; - let response = fields - .iter() - .find_map(|(name, value)| (name == "response").then(|| value.clone())) - .ok_or(Error::InvalidEntry)?; - let distance = fields - .iter() - .find_map(|(name, value)| (name == "vector_distance").then(|| value.clone())) - .ok_or(Error::InvalidEntry)?; - let distance = parse_f64(&distance)?; - if 1.0 - distance < index.similarity_threshold { - return Ok(None); - } - Ok(Some(response)) -} - -fn ensure_index( - connection: &mut ConnectionRef<'_>, - index_name: &str, - prefix: &str, - index_dimension: &Mutex>, - dimension: usize, -) -> Result<(), Error> { - if index_dimension - .lock() - .map_err(|_| Error::Unavailable)? - .is_some_and(|existing| existing == dimension) - { - return Ok(()); - } - let create = redis::cmd("FT.CREATE") - .arg(index_name) - .arg("ON") - .arg("HASH") - .arg("PREFIX") - .arg(1) - .arg(prefix) - .arg("SCHEMA") - .arg("litellm_cache_key") - .arg("TAG") - .arg("embedding") - .arg("VECTOR") - .arg("HNSW") - .arg(6) - .arg("TYPE") - .arg("FLOAT32") - .arg("DIM") - .arg(dimension) - .arg("DISTANCE_METRIC") - .arg("COSINE") - .query::(connection) - .map(|_| ()) - .map_err(|error| error.to_string()); - if let Err(message) = create { - if !message.to_ascii_lowercase().contains("already exists") { - return Err(Error::Unavailable); - } - let info = redis::cmd("FT.INFO") - .arg(index_name) - .query::(connection) - .map_err(|_| Error::Unavailable)?; - let existing = index_dimension_from_info(&info).ok_or(Error::Unavailable)?; - if existing != dimension { - return Err(Error::Unavailable); - } - } - *index_dimension.lock().map_err(|_| Error::Unavailable)? = Some(dimension); - Ok(()) -} - -fn index_dimension_from_info(value: &redis::Value) -> Option { - let redis::Value::Array(values) = value else { - return None; - }; - let attributes = values.windows(2).find_map(|pair| { - (value_text(&pair[0]).as_deref() == Some("attributes")).then_some(&pair[1]) - })?; - let redis::Value::Array(fields) = attributes else { - return None; - }; - fields.iter().find_map(|field| { - let redis::Value::Array(values) = field else { - return None; - }; - let flattened = values.iter().flat_map(|value| match value { - redis::Value::Array(values) => values.as_slice(), - _ => std::slice::from_ref(value), - }); - let values = flattened.collect::>(); - values.windows(2).find_map(|pair| { - if value_text(pair[0]).as_deref() == Some("dimensions") { - return value_text(pair[1]).and_then(|value| value.parse().ok()); - } - None - }) - }) -} - -type SearchFields = Vec<(String, Vec)>; - -fn search_fields(value: redis::Value) -> Result, Error> { - let redis::Value::Array(values) = value else { - return Err(Error::InvalidEntry); - }; - let total = parse_i64(values.first().ok_or(Error::InvalidEntry)?)?; - if total <= 0 || values.len() < 3 { - return Ok(None); - } - let redis::Value::Array(fields) = &values[2] else { - return Err(Error::InvalidEntry); - }; - let (pairs, remainder) = fields.as_chunks::<2>(); - if !remainder.is_empty() { - return Err(Error::InvalidEntry); - } - let pairs = pairs - .iter() - .map(|pair| { - Ok(( - value_text(&pair[0]).ok_or(Error::InvalidEntry)?, - value_bytes(&pair[1])?, - )) - }) - .collect::, Error>>()?; - Ok(Some(pairs)) -} - -fn parse_i64(value: &redis::Value) -> Result { - value_text(value) - .ok_or(Error::InvalidEntry)? - .parse() - .map_err(|_| Error::InvalidEntry) -} - -fn parse_f64(value: &[u8]) -> Result { - std::str::from_utf8(value) - .map_err(|_| Error::InvalidEntry)? - .parse() - .map_err(|_| Error::InvalidEntry) -} - -fn value_text(value: &redis::Value) -> Option { - match value { - redis::Value::BulkString(bytes) => String::from_utf8(bytes.clone()).ok(), - redis::Value::SimpleString(value) => Some(value.clone()), - redis::Value::Int(value) => Some(value.to_string()), - _ => None, - } -} - -fn value_bytes(value: &redis::Value) -> Result, Error> { - match value { - redis::Value::BulkString(bytes) => Ok(bytes.clone()), - redis::Value::SimpleString(value) => Ok(value.as_bytes().to_vec()), - redis::Value::Int(value) => Ok(value.to_string().into_bytes()), - _ => Err(Error::InvalidEntry), - } -} - -#[cfg(test)] -mod tests { - use std::{ - collections::VecDeque, - sync::{Arc, Mutex}, - time::Duration, - }; - - use litellm_cache::{BaseCache, CacheCodec}; - use litellm_cache_response::{ - CacheEntry, CacheKeyInput, ResponseCache, ResponseCacheCodec, ResponseCacheRequest, - }; - use redis_test::MockRedisConnection; - use rstest::rstest; - use serde_json::{Value, json}; - - use super::{ - Embedder, PreparedEmbedding, ValkeySemanticCache, ValkeySemanticConfig, - index_dimension_from_info, prompt_from_context, scope_tag, - }; - - #[derive(Clone)] - struct FixedEmbedder { - vector: Vec, - calls: EmbedderCalls, - } - - type EmbedderCalls = Arc)>>>; - type RecordingCache = - ValkeySemanticCache; - type RecordingSetup = (RecordingCache, Arc>>>, EmbedderCalls); - - impl Embedder for FixedEmbedder { - fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, super::Error> { - self.calls - .lock() - .unwrap() - .push((prompt.into(), metadata.cloned())); - Ok(self.vector.clone()) - } - - async fn async_embed( - &self, - prompt: &str, - metadata: Option<&Value>, - ) -> Result, super::Error> { - self.embed(prompt, metadata) - } - } - - struct RecordingConnection { - requests: Arc>>>, - replies: Mutex>>, - } - - impl RecordingConnection { - fn new(replies: impl IntoIterator>) -> Self { - Self { - requests: Arc::default(), - replies: Mutex::new(replies.into_iter().collect()), - } - } - - fn requests(&self) -> Arc>>> { - Arc::clone(&self.requests) - } - - fn reply(&self) -> redis::RedisResult { - self.replies - .lock() - .unwrap() - .pop_front() - .unwrap_or_else(|| Ok(redis::Value::SimpleString("OK".into()))) - } - } - - impl redis::ConnectionLike for RecordingConnection { - fn req_packed_command(&mut self, command: &[u8]) -> redis::RedisResult { - self.requests.lock().unwrap().push(command.to_vec()); - self.reply() - } - - fn req_packed_commands( - &mut self, - command: &[u8], - _offset: usize, - count: usize, - ) -> redis::RedisResult> { - self.requests.lock().unwrap().push(command.to_vec()); - (0..count).map(|_| self.reply()).collect() - } - - fn get_db(&self) -> i64 { - 0 - } - - fn check_connection(&mut self) -> bool { - true - } - - fn is_open(&self) -> bool { - true - } - } - - fn context( - messages: Option, - input: Option, - ) -> litellm_cache::SemanticCacheContext { - litellm_cache::SemanticCacheContext { - messages, - input, - ..Default::default() - } - } - - #[rstest] - #[case(json!([{"content": "hello"}]), None, Some("hello"))] - #[case(json!([{"content": [{"text": "hello"}, {"text": " world"}]}]), None, Some("hello world"))] - #[case(json!([{"content": ["raw", {"text": "hello"}]}]), None, None)] - #[case(json!([{"search_results": [{"source": "s", "title": "t", "content": [{"text": "c"}], "citations": ["x"]}]}]), None, Some(r#"stc["x"]"#))] - #[case(Value::Array(vec![]), Some(json!(" hello ")), Some("hello"))] - #[case(Value::Array(vec![]), Some(json!([{"content": "first"}, {"text": "second"}])), Some("first\nsecond"))] - #[case(Value::Array(vec![]), Some(json!(" ")), None)] - fn prompt_shapes( - #[case] messages: Value, - #[case] input: Option, - #[case] expected: Option<&str>, - ) { - assert_eq!( - prompt_from_context(&context(Some(messages), input)), - expected.map(str::to_owned) - ); - } - - #[test] - fn scope_tags_are_lowercase_sha256() { - assert_eq!( - scope_tag("key"), - "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683" - ); - } - - #[test] - fn existing_index_dimension_is_read_from_attributes() { - let info = redis::Value::Array(vec![ - redis::Value::SimpleString("attributes".into()), - redis::Value::Array(vec![redis::Value::Array(vec![ - redis::Value::SimpleString("identifier".into()), - redis::Value::SimpleString("embedding".into()), - redis::Value::Array(vec![ - redis::Value::SimpleString("dimensions".into()), - redis::Value::SimpleString("2".into()), - ]), - ])]), - ]); - assert_eq!(index_dimension_from_info(&info), Some(2)); - } - - #[tokio::test] - async fn unsupported_connection_test_is_reported() { - let cache = ValkeySemanticCache::with_connection( - MockRedisConnection::new([]).assert_all_commands_consumed(), - FixedEmbedder { - vector: vec![1.0, 0.0], - calls: Arc::default(), - }, - ResponseCacheCodec, - ValkeySemanticConfig { - similarity_threshold: 0.8, - index_name: "test".into(), - }, - ); - assert_eq!( - cache.test_connection().await, - Err(super::Error::UnsupportedOperation) - ); - } - - #[tokio::test] - async fn prepared_embedding_returns_its_vector_for_any_prompt() { - let embedding = PreparedEmbedding(vec![1.0, 2.0]); - assert_eq!( - embedding - .async_embed("different prompt", None) - .await - .unwrap(), - vec![1.0, 2.0] - ); - } - - #[test] - fn with_embedder_shares_index_state_and_connections() { - let entry = CacheEntry { - timestamp: Some(1.0), - response: json!({"answer": "ok"}), - }; - let encoded = ResponseCacheCodec.encode(&entry).unwrap(); - let cache = ValkeySemanticCache::with_connection( - RecordingConnection::new([ok(), ok(), Ok(search_hit(encoded, "0.1"))]), - FixedEmbedder { - vector: vec![1.0, 0.0], - calls: Arc::default(), - }, - ResponseCacheCodec, - ValkeySemanticConfig { - similarity_threshold: 0.8, - index_name: "test".into(), - }, - ); - cache - .set_cache("key", entry.clone(), &semantic_context(None)) - .unwrap(); - let prepared = cache.with_embedder(PreparedEmbedding(vec![1.0, 0.0])); - assert_eq!( - prepared.get_cache("key", &semantic_context(None)).unwrap(), - Some(entry) - ); - } - - #[test] - fn missing_prompt_does_not_touch_redis() { - let cache = ValkeySemanticCache::with_connection( - MockRedisConnection::new([]).assert_all_commands_consumed(), - FixedEmbedder { - vector: vec![1.0, 0.0], - calls: Arc::default(), - }, - ResponseCacheCodec, - ValkeySemanticConfig { - similarity_threshold: 0.8, - index_name: "test".into(), - }, - ); - assert_eq!(cache.get_cache("key", &context(None, None)).unwrap(), None); - assert_eq!(cache.get_ttl(&context(None, None)), None); - } - - fn semantic_context(ttl: Option) -> litellm_cache::SemanticCacheContext { - litellm_cache::SemanticCacheContext { - messages: Some(json!([{"role": "user", "content": "hello"}])), - metadata: Some(json!({"source": "test"})), - ttl, - ..Default::default() - } - } - - fn cache_with_recording( - replies: impl IntoIterator>, - vector: Vec, - threshold: f64, - ) -> RecordingSetup { - let connection = RecordingConnection::new(replies); - let requests = connection.requests(); - let calls: EmbedderCalls = Arc::default(); - let cache = ValkeySemanticCache::with_connection( - connection, - FixedEmbedder { - vector, - calls: Arc::clone(&calls), - }, - ResponseCacheCodec, - ValkeySemanticConfig { - similarity_threshold: threshold, - index_name: "test".into(), - }, - ); - (cache, requests, calls) - } - - fn ok() -> redis::RedisResult { - Ok(redis::Value::SimpleString("OK".into())) - } - - fn already_exists() -> redis::RedisResult { - Err(redis::RedisError::from(( - redis::ErrorKind::Io, - "already exists", - ))) - } - - fn info_dimension(dimension: usize) -> redis::Value { - redis::Value::Array(vec![ - redis::Value::SimpleString("attributes".into()), - redis::Value::Array(vec![redis::Value::Array(vec![ - redis::Value::SimpleString("embedding".into()), - redis::Value::Array(vec![ - redis::Value::SimpleString("dimensions".into()), - redis::Value::Int(dimension as i64), - ]), - ])]), - ]) - } - - fn search_hit(response: Vec, distance: &str) -> redis::Value { - redis::Value::Array(vec![ - redis::Value::Int(1), - redis::Value::BulkString(b"test:document".to_vec()), - redis::Value::Array(vec![ - redis::Value::BulkString(b"response".to_vec()), - redis::Value::BulkString(response), - redis::Value::BulkString(b"vector_distance".to_vec()), - redis::Value::BulkString(distance.as_bytes().to_vec()), - ]), - ]) - } - - fn requests_text(requests: &Arc>>>) -> String { - requests - .lock() - .unwrap() - .iter() - .map(|request| String::from_utf8_lossy(request)) - .collect::>() - .join("\n") - } - - #[test] - fn set_without_ttl_writes_hset_without_expire() { - let (cache, requests, calls) = cache_with_recording([ok()], vec![1.0, 0.0], 0.8); - cache - .set_cache( - "key", - CacheEntry { - timestamp: None, - response: json!({"answer": "ok"}), - }, - &semantic_context(None), - ) - .unwrap(); - let text = requests_text(&requests); - assert!(text.contains("FT.CREATE")); - assert!(text.contains("HSET")); - assert!( - text.contains("test:2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683:") - ); - assert!(!text.contains("EXPIRE")); - assert_eq!( - *calls.lock().unwrap(), - vec![("hello".into(), Some(json!({"source": "test"})))] - ); - } - - #[test] - fn set_with_ttl_truncates_expire_seconds() { - let (cache, requests, _) = cache_with_recording([ok()], vec![1.0, 0.0], 0.8); - cache - .set_cache( - "key", - CacheEntry { - timestamp: None, - response: json!({"answer": "ok"}), - }, - &semantic_context(Some(Duration::from_millis(1900))), - ) - .unwrap(); - let text = requests_text(&requests); - assert!(text.contains("EXPIRE")); - assert!(text.contains("\r\n$1\r\n1\r\n")); - } - - #[test] - fn second_set_skips_create_after_dimension_is_cached() { - let (cache, requests, _) = cache_with_recording([ok()], vec![1.0, 0.0], 0.8); - let context = semantic_context(None); - let entry = CacheEntry { - timestamp: None, - response: json!({"answer": "ok"}), - }; - cache.set_cache("key", entry.clone(), &context).unwrap(); - cache.set_cache("key", entry, &context).unwrap(); - let text = requests_text(&requests); - assert_eq!(text.matches("FT.CREATE").count(), 1); - assert_eq!(text.matches("HSET").count(), 2); - } - - #[test] - fn existing_index_dimension_must_match_embedding() { - let (cache, _, _) = cache_with_recording( - [already_exists(), Ok(info_dimension(2))], - vec![1.0, 0.0], - 0.8, - ); - cache - .set_cache( - "key", - CacheEntry { - timestamp: None, - response: json!({"answer": "ok"}), - }, - &semantic_context(None), - ) - .unwrap(); - - let (cache, _, _) = cache_with_recording( - [already_exists(), Ok(info_dimension(3))], - vec![1.0, 0.0], - 0.8, - ); - assert_eq!( - cache.set_cache( - "key", - CacheEntry { - timestamp: None, - response: json!({"answer": "ok"}), - }, - &semantic_context(None), - ), - Err(super::Error::Unavailable) - ); - } - - #[test] - fn get_applies_threshold_and_decodes_entry() { - let entry = CacheEntry { - timestamp: Some(1.0), - response: json!({"answer": "ok"}), - }; - let encoded = ResponseCacheCodec.encode(&entry).unwrap(); - let (cache, _, _) = cache_with_recording( - [ok(), Ok(search_hit(encoded.clone(), "0.1"))], - vec![1.0, 0.0], - 0.8, - ); - assert_eq!( - cache.get_cache("key", &semantic_context(None)).unwrap(), - Some(entry) - ); - - let (cache, _, _) = - cache_with_recording([ok(), Ok(search_hit(encoded, "0.5"))], vec![1.0, 0.0], 0.8); - assert_eq!( - cache.get_cache("key", &semantic_context(None)).unwrap(), - None - ); - } - - #[test] - fn get_zero_docs_is_a_miss() { - let (cache, _, _) = cache_with_recording( - [ok(), Ok(redis::Value::Array(vec![redis::Value::Int(0)]))], - vec![1.0, 0.0], - 0.8, - ); - assert_eq!( - cache.get_cache("key", &semantic_context(None)).unwrap(), - None - ); - } - - #[rstest] - #[case(redis::Value::Array(vec![ - redis::Value::Int(1), - redis::Value::BulkString(b"document".to_vec()), - redis::Value::Array(vec![ - redis::Value::BulkString(b"vector_distance".to_vec()), - redis::Value::BulkString(b"0.1".to_vec()), - ]), - ]))] - #[case(redis::Value::Array(vec![ - redis::Value::Int(1), - redis::Value::BulkString(b"document".to_vec()), - redis::Value::Array(vec![ - redis::Value::BulkString(b"response".to_vec()), - redis::Value::BulkString(b"not-json".to_vec()), - redis::Value::BulkString(b"vector_distance".to_vec()), - redis::Value::BulkString(b"abc".to_vec()), - ]), - ]))] - fn malformed_entries_are_invalid(#[case] search: redis::Value) { - let (cache, _, _) = cache_with_recording([ok(), Ok(search)], vec![1.0, 0.0], 0.8); - assert_eq!( - cache.get_cache("key", &semantic_context(None)), - Err(super::Error::InvalidEntry) - ); - } - - #[test] - fn response_cache_turns_invalid_entries_into_misses() { - let (cache, _, _) = cache_with_recording( - [ - ok(), - Ok(redis::Value::Array(vec![ - redis::Value::Int(1), - redis::Value::BulkString(b"document".to_vec()), - redis::Value::Array(vec![ - redis::Value::BulkString(b"response".to_vec()), - redis::Value::BulkString(b"not-json".to_vec()), - redis::Value::BulkString(b"vector_distance".to_vec()), - redis::Value::BulkString(b"0.1".to_vec()), - ]), - ])), - ], - vec![1.0, 0.0], - 0.8, - ); - let service = ResponseCache::new(Arc::new(cache)); - let request = ResponseCacheRequest { - key: CacheKeyInput { - preset: Some("key".into()), - ..Default::default() - }, - context: semantic_context(None), - ..ResponseCacheRequest::new(CacheKeyInput::default()) - }; - assert_eq!(service.lookup(&request, Duration::ZERO).unwrap(), None); - } - - #[tokio::test] - async fn async_set_and_get_use_shared_document_helpers() { - let entry = CacheEntry { - timestamp: Some(1.0), - response: json!({"answer": "ok"}), - }; - let encoded = ResponseCacheCodec.encode(&entry).unwrap(); - let (cache, requests, calls) = cache_with_recording( - [ok(), ok(), ok(), Ok(search_hit(encoded, "0.1"))], - vec![1.0, 0.0], - 0.8, - ); - let context = semantic_context(Some(Duration::from_millis(1900))); - cache - .async_set_cache("key", entry.clone(), context.clone()) - .await - .unwrap(); - assert_eq!( - cache.async_get_cache("key", &context).await.unwrap(), - Some(entry) - ); - let text = requests_text(&requests); - assert!(text.contains("FT.CREATE")); - assert!(text.contains("HSET")); - assert!(text.contains("EXPIRE")); - assert_eq!(calls.lock().unwrap().len(), 2); - } -} +pub use cache::ValkeySemanticCache; +pub use config::{DEFAULT_INDEX_NAME, ValkeySemanticConfig}; diff --git a/litellm-rust/crates/cache-valkey-semantic/src/search.rs b/litellm-rust/crates/cache-valkey-semantic/src/search.rs new file mode 100644 index 00000000000..3b4fae84317 --- /dev/null +++ b/litellm-rust/crates/cache-valkey-semantic/src/search.rs @@ -0,0 +1,163 @@ +use std::time::Duration; + +use litellm_cache::{Error, semantic::SemanticLookup}; +use litellm_cache_redis::connection::ConnectionRef; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use crate::index::{IndexState, ensure_index}; + +/// `_scope_tag`: valkey-search TAG fields cannot match arbitrary keys verbatim, so scopes are +/// the key's lowercase SHA-256. +pub(crate) fn scope_tag(key: &str) -> String { + let digest = Sha256::digest(key.as_bytes()); + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} + +pub(crate) fn embedding_bytes(embedding: &[f32]) -> Vec { + embedding + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect() +} + +/// `HSET` a fresh `:` document, then `EXPIRE` it when a TTL is set. +pub(crate) fn write_document( + connection: &mut ConnectionRef<'_>, + index: &IndexState, + scope: &str, + prompt: &str, + response: Vec, + vector: Vec, + ttl: Option, +) -> Result<(), Error> { + ensure_index(connection, index, vector.len() / size_of::())?; + let document = format!("{}{scope}:{}", index.prefix, Uuid::new_v4()); + let mut pipeline = redis::pipe(); + pipeline + .cmd("HSET") + .arg(&document) + .arg("litellm_cache_key") + .arg(scope) + .arg("prompt") + .arg(prompt) + .arg("response") + .arg(response) + .arg("embedding") + .arg(vector) + .ignore(); + if let Some(ttl) = ttl { + pipeline + .cmd("EXPIRE") + .arg(&document) + .arg(ttl.as_secs()) + .ignore(); + } + pipeline + .query::<()>(connection) + .map_err(|_| Error::Unavailable) +} + +/// The KNN-1 search within `scope`: the closest document's similarity, and its stored response +/// when that similarity reaches the threshold. No document reads as a similarity of `0.0`. +pub(crate) fn search_document( + connection: &mut ConnectionRef<'_>, + index: &IndexState, + scope: &str, + vector: Vec, +) -> Result>, Error> { + ensure_index(connection, index, vector.len() / size_of::())?; + let query = + format!("(@litellm_cache_key:{{{scope}}})=>[KNN 1 @embedding $vec AS vector_distance]"); + let response = redis::cmd("FT.SEARCH") + .arg(&index.name) + .arg(query) + .arg("PARAMS") + .arg(2) + .arg("vec") + .arg(vector) + .arg("RETURN") + .arg(2) + .arg("response") + .arg("vector_distance") + .arg("DIALECT") + .arg(2) + .query::(connection) + .map_err(|_| Error::Unavailable)?; + let Some(fields) = search_fields(response)? else { + return Ok(SemanticLookup::miss(Some(0.0))); + }; + let field = |name: &str| { + fields + .iter() + .find_map(|(field, value)| (field == name).then(|| value.clone())) + .ok_or(Error::InvalidEntry) + }; + let response = field("response")?; + let similarity = 1.0 - parse_f64(&field("vector_distance")?)?; + Ok(SemanticLookup { + value: (similarity >= index.similarity_threshold).then_some(response), + similarity: Some(similarity), + }) +} + +type SearchFields = Vec<(String, Vec)>; + +fn search_fields(value: redis::Value) -> Result, Error> { + let redis::Value::Array(values) = value else { + return Err(Error::InvalidEntry); + }; + let total = parse_i64(values.first().ok_or(Error::InvalidEntry)?)?; + if total <= 0 || values.len() < 3 { + return Ok(None); + } + let redis::Value::Array(fields) = &values[2] else { + return Err(Error::InvalidEntry); + }; + let (pairs, remainder) = fields.as_chunks::<2>(); + if !remainder.is_empty() { + return Err(Error::InvalidEntry); + } + pairs + .iter() + .map(|pair| { + Ok(( + value_text(&pair[0]).ok_or(Error::InvalidEntry)?, + value_bytes(&pair[1])?, + )) + }) + .collect::, Error>>() + .map(Some) +} + +fn parse_i64(value: &redis::Value) -> Result { + value_text(value) + .ok_or(Error::InvalidEntry)? + .parse() + .map_err(|_| Error::InvalidEntry) +} + +fn parse_f64(value: &[u8]) -> Result { + std::str::from_utf8(value) + .map_err(|_| Error::InvalidEntry)? + .parse() + .map_err(|_| Error::InvalidEntry) +} + +pub(crate) fn value_text(value: &redis::Value) -> Option { + match value { + redis::Value::BulkString(bytes) => String::from_utf8(bytes.clone()).ok(), + redis::Value::SimpleString(value) => Some(value.clone()), + redis::Value::Int(value) => Some(value.to_string()), + _ => None, + } +} + +fn value_bytes(value: &redis::Value) -> Result, Error> { + match value { + redis::Value::BulkString(bytes) => Ok(bytes.clone()), + redis::Value::SimpleString(value) => Ok(value.as_bytes().to_vec()), + redis::Value::Int(value) => Ok(value.to_string().into_bytes()), + _ => Err(Error::InvalidEntry), + } +} diff --git a/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs b/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs new file mode 100644 index 00000000000..1c693d0875d --- /dev/null +++ b/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs @@ -0,0 +1,417 @@ +mod support; + +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; + +use litellm_cache::{ + BaseCache, Error, JsonCodec, SemanticCacheContext, + semantic::{PreparedEmbedding, SemanticCache, SemanticLookup}, +}; +use litellm_cache_valkey_semantic::{ + DEFAULT_INDEX_NAME, ValkeySemanticCache, ValkeySemanticConfig, +}; +use redis_test::MockRedisConnection; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; +use support::{EmbedCalls, FakeEmbedder, RecordingConnection}; + +type Requests = Arc>>>; +type RecordingCache = ValkeySemanticCache, RecordingConnection>; + +const KEY_SCOPE: &str = "2c70e12b7a0646f92279f427c7b38e7334d8e5389cff167a1dc30e73f826b683"; + +struct Recording { + cache: RecordingCache, + requests: Requests, + calls: EmbedCalls, +} + +impl Recording { + fn text(&self) -> String { + self.requests + .lock() + .unwrap() + .iter() + .map(|request| String::from_utf8_lossy(request).into_owned()) + .collect::>() + .join("\n") + } +} + +fn config() -> ValkeySemanticConfig { + ValkeySemanticConfig { + similarity_threshold: 0.8, + index_name: "test".into(), + } +} + +fn recording(replies: impl IntoIterator>) -> Recording { + let connection = RecordingConnection::new(replies); + let requests = connection.requests(); + let embedder = FakeEmbedder::new(&[]); + let calls = Arc::clone(&embedder.calls); + Recording { + cache: ValkeySemanticCache::with_connection( + connection, + embedder, + JsonCodec::new(), + config(), + ), + requests, + calls, + } +} + +#[fixture] +fn entry() -> Value { + json!({"timestamp": 1.0, "response": {"answer": "ok"}}) +} + +#[fixture] +fn context() -> SemanticCacheContext { + SemanticCacheContext { + messages: Some(json!([{"role": "user", "content": "hello"}])), + metadata: Some(json!({"source": "test"})), + ..Default::default() + } +} + +fn ok() -> redis::RedisResult { + Ok(redis::Value::SimpleString("OK".into())) +} + +fn already_exists() -> redis::RedisResult { + Err(redis::RedisError::from(( + redis::ErrorKind::Io, + "already exists", + ))) +} + +fn bulk(value: &[u8]) -> redis::Value { + redis::Value::BulkString(value.to_vec()) +} + +fn search_hit(response: &[u8], distance: &str) -> redis::Value { + redis::Value::Array(vec![ + redis::Value::Int(1), + bulk(b"test:document"), + redis::Value::Array(vec![ + bulk(b"response"), + bulk(response), + bulk(b"vector_distance"), + bulk(distance.as_bytes()), + ]), + ]) +} + +fn encoded(value: &Value) -> Vec { + serde_json::to_vec(value).unwrap() +} + +/// `FT.INFO` with the vector field's dimension nested one level down, as valkey-search reports. +fn nested_dimension_info(dimension: i64) -> redis::Value { + redis::Value::Array(vec![ + redis::Value::SimpleString("attributes".into()), + redis::Value::Array(vec![redis::Value::Array(vec![ + redis::Value::SimpleString("embedding".into()), + redis::Value::Array(vec![ + redis::Value::SimpleString("dimensions".into()), + redis::Value::Int(dimension), + ]), + ])]), + ]) +} + +/// `FT.INFO` with `dimensions` as a sibling string of the identifier. +fn flat_dimension_info(dimension: &str) -> redis::Value { + redis::Value::Array(vec![ + redis::Value::SimpleString("attributes".into()), + redis::Value::Array(vec![redis::Value::Array(vec![ + redis::Value::SimpleString("identifier".into()), + redis::Value::SimpleString("embedding".into()), + redis::Value::Array(vec![ + redis::Value::SimpleString("dimensions".into()), + redis::Value::SimpleString(dimension.into()), + ]), + ])]), + ]) +} + +#[rstest] +#[case::string_content(json!([{"content": "hello"}]), None, Some("hello"))] +#[case::text_parts(json!([{"content": [{"text": "hello"}, {"text": " world"}]}]), None, Some("hello world"))] +#[case::non_object_parts_are_skipped(json!([{"content": ["raw", {"text": "hello"}]}]), None, Some("hello"))] +#[case::search_results(json!([{"search_results": [{"source": "s", "title": "t", "content": [{"text": "c"}], "citations": ["x"]}]}]), None, Some(r#"stc["x"]"#))] +#[case::responses_string_input(json!([]), Some(json!(" hello ")), Some("hello"))] +#[case::responses_item_input(json!([]), Some(json!([{"content": "first"}, {"text": "second"}])), Some("first\nsecond"))] +#[case::blank_input(json!([]), Some(json!(" ")), None)] +fn prompt_shapes_follow_redis_semantic_extraction( + #[case] messages: Value, + #[case] input: Option, + #[case] expected: Option<&str>, +) { + let recording = recording([ok(), Ok(redis::Value::Array(vec![redis::Value::Int(0)]))]); + let context = SemanticCacheContext { + messages: Some(messages), + input, + ..Default::default() + }; + + assert_eq!(recording.cache.get_cache("key", &context).unwrap(), None); + + let prompts = recording + .calls + .lock() + .unwrap() + .iter() + .map(|(prompt, _)| prompt.clone()) + .collect::>(); + assert_eq!( + prompts, + expected.into_iter().map(str::to_owned).collect::>() + ); + assert_eq!( + recording.requests.lock().unwrap().is_empty(), + expected.is_none() + ); +} + +#[rstest] +#[case::key("key", KEY_SCOPE)] +#[case::empty_key("", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")] +fn documents_are_scoped_by_the_keys_sha256( + #[case] key: &str, + #[case] scope: &str, + entry: Value, + context: SemanticCacheContext, +) { + let recording = recording([ok()]); + + recording.cache.set_cache(key, entry, &context).unwrap(); + + let text = recording.text(); + assert!(text.contains(&format!("test:{scope}:"))); + assert!(text.contains(&format!("litellm_cache_key\r\n$64\r\n{scope}\r\n"))); +} + +#[rstest] +#[case::no_ttl(None, None)] +#[case::whole_seconds(Some(Duration::from_secs(5)), Some("5"))] +#[case::fractional_seconds_truncate(Some(Duration::from_millis(1900)), Some("1"))] +fn set_writes_hset_and_expires_only_with_a_ttl( + #[case] ttl: Option, + #[case] expire: Option<&str>, + entry: Value, + context: SemanticCacheContext, +) { + let recording = recording([ok()]); + + recording + .cache + .set_cache( + "key", + entry, + &SemanticCacheContext { + ttl, + ..context.clone() + }, + ) + .unwrap(); + + let text = recording.text(); + assert!(text.contains("FT.CREATE")); + assert!(text.contains("HSET")); + let expire_seconds = text + .split_once("EXPIRE\r\n") + .and_then(|(_, rest)| rest.split("\r\n").nth(3)); + assert_eq!(expire_seconds, expire); + assert_eq!( + *recording.calls.lock().unwrap(), + vec![("hello".to_owned(), context.metadata)] + ); +} + +#[rstest] +fn second_set_skips_create_after_dimension_is_cached(entry: Value, context: SemanticCacheContext) { + let recording = recording([ok()]); + + recording + .cache + .set_cache("key", entry.clone(), &context) + .unwrap(); + recording.cache.set_cache("key", entry, &context).unwrap(); + + let text = recording.text(); + assert_eq!(text.matches("FT.CREATE").count(), 1); + assert_eq!(text.matches("HSET").count(), 2); +} + +#[rstest] +#[case::nested_matching(nested_dimension_info(3), Ok(()))] +#[case::nested_mismatch(nested_dimension_info(2), Err(Error::Unavailable))] +#[case::flat_matching(flat_dimension_info("3"), Ok(()))] +#[case::flat_mismatch(flat_dimension_info("2"), Err(Error::Unavailable))] +#[case::unreported_dimension_is_accepted(redis::Value::Array(vec![]), Ok(()))] +fn existing_index_dimension_must_match_embedding( + #[case] info: redis::Value, + #[case] expected: Result<(), Error>, + entry: Value, + context: SemanticCacheContext, +) { + let recording = recording([already_exists(), Ok(info)]); + + assert_eq!(recording.cache.set_cache("key", entry, &context), expected); +} + +#[rstest] +#[case::create_failure(Err(redis::RedisError::from((redis::ErrorKind::Io, "boom"))))] +fn index_creation_failures_are_unavailable( + #[case] reply: redis::RedisResult, + entry: Value, + context: SemanticCacheContext, +) { + let recording = recording([reply]); + + assert_eq!( + recording.cache.set_cache("key", entry, &context), + Err(Error::Unavailable) + ); +} + +#[rstest] +#[case::within_threshold(search_hit(&encoded(&entry()), "0.1"), Ok(Some(entry())))] +#[case::at_threshold(search_hit(&encoded(&entry()), "0.2"), Ok(Some(entry())))] +#[case::beyond_threshold(search_hit(&encoded(&entry()), "0.5"), Ok(None))] +#[case::zero_documents(redis::Value::Array(vec![redis::Value::Int(0)]), Ok(None))] +#[case::missing_response( + redis::Value::Array(vec![ + redis::Value::Int(1), + bulk(b"document"), + redis::Value::Array(vec![bulk(b"vector_distance"), bulk(b"0.1")]), + ]), + Err(Error::InvalidEntry) +)] +#[case::unparsable_distance(search_hit(b"not-json", "abc"), Err(Error::InvalidEntry))] +#[case::undecodable_response(search_hit(b"not-json", "0.1"), Err(Error::InvalidEntry))] +fn get_applies_threshold_and_decodes_entry( + #[case] reply: redis::Value, + #[case] expected: Result, Error>, + context: SemanticCacheContext, +) { + let recording = recording([ok(), Ok(reply)]); + + assert_eq!(recording.cache.get_cache("key", &context), expected); +} + +#[rstest] +#[case::hit(context(), Some(search_hit(&encoded(&entry()), "0.1")), Some(entry()), Some(1.0 - 0.1))] +#[case::below_threshold(context(), Some(search_hit(&encoded(&entry()), "0.5")), None, Some(1.0 - 0.5))] +#[case::no_results(context(), Some(redis::Value::Array(vec![redis::Value::Int(0)])), None, Some(0.0))] +#[case::no_prompt(SemanticCacheContext::default(), None, None, Some(0.0))] +#[tokio::test] +async fn lookup_reports_python_semantic_similarity( + #[case] context: SemanticCacheContext, + #[case] reply: Option, + #[case] value: Option, + #[case] similarity: Option, + #[values(false, true)] use_async: bool, +) { + let searched = reply.is_some(); + let recording = recording(reply.map_or_else(Vec::new, |reply| vec![ok(), Ok(reply)])); + + let lookup = if use_async { + recording + .cache + .async_get_cache_with_similarity("key", &context) + .await + } else { + recording.cache.get_cache_with_similarity("key", &context) + }; + + assert_eq!(lookup, Ok(SemanticLookup { value, similarity })); + assert_eq!(recording.text().contains("FT.SEARCH"), searched); +} + +#[rstest] +#[tokio::test] +async fn missing_prompt_does_not_touch_valkey(entry: Value) { + let cache = ValkeySemanticCache::with_connection( + MockRedisConnection::new([]).assert_all_commands_consumed(), + FakeEmbedder::new(&[]), + JsonCodec::new(), + config(), + ); + let context = SemanticCacheContext::default(); + + cache.set_cache("key", entry.clone(), &context).unwrap(); + assert_eq!(cache.get_cache("key", &context).unwrap(), None); + cache + .async_set_cache("key", entry, context.clone()) + .await + .unwrap(); + assert_eq!(cache.async_get_cache("key", &context).await.unwrap(), None); + assert_eq!(cache.get_ttl(&context), None); +} + +#[rstest] +fn with_embedder_shares_index_state_and_connections(entry: Value, context: SemanticCacheContext) { + let recording = recording([ok(), ok(), Ok(search_hit(&encoded(&entry), "0.1"))]); + recording + .cache + .set_cache("key", entry.clone(), &context) + .unwrap(); + + let prepared = recording + .cache + .with_embedder(PreparedEmbedding(vec![0.1, 0.2, 0.3])); + + assert_eq!(prepared.get_cache("key", &context).unwrap(), Some(entry)); + assert_eq!(recording.text().matches("FT.CREATE").count(), 1); +} + +#[rstest] +fn accessors_report_the_config() { + let recording = recording([]); + + assert_eq!(recording.cache.index_name(), "test"); + assert_eq!(recording.cache.similarity_threshold(), 0.8); + assert_eq!(DEFAULT_INDEX_NAME, "litellm_semantic_cache_index"); +} + +#[rstest] +#[tokio::test] +async fn async_set_and_get_use_shared_document_helpers( + entry: Value, + context: SemanticCacheContext, +) { + let recording = recording([ok(), ok(), ok(), Ok(search_hit(&encoded(&entry), "0.1"))]); + let context = SemanticCacheContext { + ttl: Some(Duration::from_millis(1900)), + ..context + }; + + recording + .cache + .async_set_cache("key", entry.clone(), context.clone()) + .await + .unwrap(); + assert_eq!( + recording + .cache + .async_get_cache("key", &context) + .await + .unwrap(), + Some(entry) + ); + + let text = recording.text(); + assert!(text.contains("FT.CREATE")); + assert!(text.contains("HSET")); + assert!(text.contains("EXPIRE")); + assert_eq!( + *recording.calls.lock().unwrap(), + vec![("hello".to_owned(), context.metadata.clone()); 2] + ); +} diff --git a/litellm-rust/crates/cache-valkey-semantic/tests/contract.rs b/litellm-rust/crates/cache-valkey-semantic/tests/contract.rs new file mode 100644 index 00000000000..71081fca92b --- /dev/null +++ b/litellm-rust/crates/cache-valkey-semantic/tests/contract.rs @@ -0,0 +1,62 @@ +//! `overwrite_replaces` does not apply: like Python, every write is a new `:` +//! document, so a second write with the same prompt adds a tie instead of replacing the first. + +mod support; + +use litellm_cache::{JsonCodec, SemanticCacheContext, semantic::PreparedEmbedding}; +use litellm_cache_testing as contract; +use litellm_cache_valkey_semantic::{ + DEFAULT_INDEX_NAME, ValkeySemanticCache, ValkeySemanticConfig, +}; +use rstest::{fixture, rstest}; +use serde_json::{Value, json}; +use support::FakeSearch; + +type Cache = ValkeySemanticCache, FakeSearch>; + +const PREFIX: &str = "contract:"; + +#[fixture] +fn cache() -> Cache { + ValkeySemanticCache::with_connection( + FakeSearch::default(), + PreparedEmbedding(vec![0.6, 0.8]), + JsonCodec::new(), + ValkeySemanticConfig { + similarity_threshold: 0.9, + index_name: DEFAULT_INDEX_NAME.into(), + }, + ) +} + +#[fixture] +fn context() -> SemanticCacheContext { + SemanticCacheContext { + messages: Some(json!([{"role": "user", "content": "contract prompt"}])), + ..Default::default() + } +} + +#[rstest] +#[tokio::test] +async fn hit_and_miss(cache: Cache, context: SemanticCacheContext) { + contract::hit_and_miss(&cache, context, PREFIX, json!({"answer": 42})).await; +} + +#[rstest] +#[tokio::test] +async fn sync_async_equivalence(cache: Cache, context: SemanticCacheContext) { + contract::sync_async_equivalence(&cache, context, PREFIX, json!("first"), json!([2])).await; +} + +#[rstest] +#[tokio::test] +async fn pipeline_writes_every_entry(cache: Cache, context: SemanticCacheContext) { + contract::pipeline_writes_every_entry( + &cache, + context, + PREFIX, + vec![json!("a"), json!(2), json!({"c": true})], + ) + .await; +} diff --git a/litellm-rust/crates/cache-valkey-semantic/tests/support/mod.rs b/litellm-rust/crates/cache-valkey-semantic/tests/support/mod.rs new file mode 100644 index 00000000000..2c6ff06b134 --- /dev/null +++ b/litellm-rust/crates/cache-valkey-semantic/tests/support/mod.rs @@ -0,0 +1,349 @@ +#![allow(dead_code)] + +use std::{ + collections::{BTreeMap, HashMap, VecDeque}, + sync::{Arc, Mutex}, +}; + +use litellm_cache::{Error, semantic::Embedder}; +use serde_json::Value; + +pub type EmbedCalls = Arc)>>>; + +/// Embeds known prompts to fixed vectors, anything else to `[0.1, 0.2, 0.3]`, and records every +/// prompt with its metadata. +pub struct FakeEmbedder { + vectors: HashMap>, + pub calls: EmbedCalls, +} + +impl FakeEmbedder { + pub fn new(vectors: &[(&str, &[f32])]) -> Self { + Self { + vectors: vectors + .iter() + .map(|(prompt, vector)| ((*prompt).to_owned(), vector.to_vec())) + .collect(), + calls: EmbedCalls::default(), + } + } +} + +impl Embedder for FakeEmbedder { + fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { + self.calls + .lock() + .unwrap() + .push((prompt.to_owned(), metadata.cloned())); + Ok(self + .vectors + .get(prompt) + .cloned() + .unwrap_or_else(|| vec![0.1, 0.2, 0.3])) + } + + async fn async_embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { + self.embed(prompt, metadata) + } +} + +struct FakeIndex { + prefix: Vec, + dims: usize, + vector_field: String, +} + +#[derive(Default)] +struct SearchState { + indexes: HashMap, + hashes: BTreeMap, BTreeMap>>, +} + +/// An in-memory valkey-search speaking the `FT.*`, `HSET` and `EXPIRE` subset the semantic cache +/// sends, with exact cosine KNN over the hashes under an index prefix. +#[derive(Clone, Default)] +pub struct FakeSearch { + state: Arc>, +} + +impl FakeSearch { + fn run(&self, args: Vec>) -> redis::RedisResult { + let mut state = self.state.lock().unwrap(); + let text = |index: usize| String::from_utf8_lossy(&args[index]).into_owned(); + match text(0).to_uppercase().as_str() { + "FT.CREATE" => { + let name = text(1); + if state.indexes.contains_key(&name) { + return Err(error("Index already exists")); + } + let position = |token: &str| args.iter().position(|arg| arg == token.as_bytes()); + let prefix = args[position("PREFIX").unwrap() + 2].clone(); + let dims = text(position("DIM").unwrap() + 1).parse().unwrap(); + let vector_field = text(position("VECTOR").unwrap() - 1); + state.indexes.insert( + name, + FakeIndex { + prefix, + dims, + vector_field, + }, + ); + Ok(redis::Value::Okay) + } + "FT.INFO" => { + let index = state + .indexes + .get(&text(1)) + .ok_or_else(|| error("Unknown index name"))?; + Ok(index_info(index)) + } + "FT.DROPINDEX" => { + state.indexes.remove(&text(1)); + Ok(redis::Value::Okay) + } + "HSET" => { + let hash = state.hashes.entry(args[1].clone()).or_default(); + for pair in args[2..].chunks(2) { + hash.insert( + String::from_utf8_lossy(&pair[0]).into_owned(), + pair[1].clone(), + ); + } + Ok(redis::Value::Int(((args.len() - 2) / 2) as i64)) + } + "EXPIRE" => Ok(redis::Value::Int(i64::from( + state.hashes.contains_key(&args[1]), + ))), + "FT.SEARCH" => { + let index = state + .indexes + .get(&text(1)) + .ok_or_else(|| error("no such index"))?; + let query = text(2); + let tag = query_tag(&query); + let params = args.iter().position(|arg| arg == b"PARAMS").unwrap(); + let vector = floats(&args[params + 3]); + let best = state + .hashes + .iter() + .filter(|(key, _)| key.starts_with(&index.prefix)) + .filter(|(_, fields)| { + fields.get("litellm_cache_key").map(Vec::as_slice) == Some(tag.as_bytes()) + }) + .filter_map(|(key, fields)| { + let stored = floats(fields.get(&index.vector_field)?); + (stored.len() == index.dims) + .then(|| (key, fields, 1.0 - cosine(&vector, &stored))) + }) + .min_by(|left, right| left.2.total_cmp(&right.2)); + let Some((key, fields, distance)) = best else { + return Ok(redis::Value::Array(vec![redis::Value::Int(0)])); + }; + let mut reply = fields + .iter() + .filter(|(name, _)| **name != index.vector_field) + .flat_map(|(name, value)| [bulk(name.as_bytes()), bulk(value)]) + .collect::>(); + reply.extend([ + bulk(b"vector_distance"), + bulk(distance.to_string().as_bytes()), + ]); + Ok(redis::Value::Array(vec![ + redis::Value::Int(1), + bulk(key), + redis::Value::Array(reply), + ])) + } + "PING" => Ok(redis::Value::SimpleString("PONG".into())), + _ => Err(error("unsupported command")), + } + } +} + +impl redis::ConnectionLike for FakeSearch { + fn req_packed_command(&mut self, command: &[u8]) -> redis::RedisResult { + let mut commands = parse_commands(command); + self.run(commands.remove(0)) + } + + fn req_packed_commands( + &mut self, + commands: &[u8], + offset: usize, + count: usize, + ) -> redis::RedisResult> { + let replies = parse_commands(commands) + .into_iter() + .map(|args| self.run(args)) + .collect::>>()?; + Ok(replies.into_iter().skip(offset).take(count).collect()) + } + + fn get_db(&self) -> i64 { + 0 + } + + fn check_connection(&mut self) -> bool { + true + } + + fn is_open(&self) -> bool { + true + } +} + +fn error(message: &'static str) -> redis::RedisError { + redis::RedisError::from((redis::ErrorKind::Extension, message)) +} + +fn bulk(bytes: &[u8]) -> redis::Value { + redis::Value::BulkString(bytes.to_vec()) +} + +fn index_info(index: &FakeIndex) -> redis::Value { + redis::Value::Array(vec![ + bulk(b"index_name"), + bulk(b"fake"), + bulk(b"attributes"), + redis::Value::Array(vec![ + redis::Value::Array(vec![ + bulk(b"identifier"), + bulk(b"litellm_cache_key"), + bulk(b"type"), + bulk(b"TAG"), + ]), + redis::Value::Array(vec![ + bulk(b"identifier"), + bulk(index.vector_field.as_bytes()), + bulk(b"type"), + bulk(b"VECTOR"), + bulk(b"index"), + redis::Value::Array(vec![ + bulk(b"dimensions"), + redis::Value::Int(index.dims as i64), + ]), + ]), + ]), + ]) +} + +/// The tag inside `@litellm_cache_key:{...}`, with query escapes removed. +fn query_tag(query: &str) -> String { + let start = query.find("@litellm_cache_key:{").unwrap() + "@litellm_cache_key:{".len(); + let mut tag = String::new(); + let mut characters = query[start..].chars(); + while let Some(character) = characters.next() { + match character { + '\\' => tag.extend(characters.next()), + '}' => break, + character => tag.push(character), + } + } + tag +} + +fn floats(bytes: &[u8]) -> Vec { + bytes + .as_chunks::<4>() + .0 + .iter() + .map(|chunk| f32::from_le_bytes(*chunk)) + .collect() +} + +fn cosine(left: &[f32], right: &[f32]) -> f64 { + let dot = left + .iter() + .zip(right) + .map(|(left, right)| f64::from(*left) * f64::from(*right)) + .sum::(); + let norm = |vector: &[f32]| { + vector + .iter() + .map(|value| f64::from(*value).powi(2)) + .sum::() + .sqrt() + }; + dot / (norm(left) * norm(right)) +} + +/// Splits a packed RESP request into each command's arguments. +fn parse_commands(mut bytes: &[u8]) -> Vec>> { + let line = |bytes: &mut &[u8]| { + let end = bytes + .windows(2) + .position(|window| window == b"\r\n") + .unwrap(); + let text = String::from_utf8(bytes[1..end].to_vec()).unwrap(); + *bytes = &bytes[end + 2..]; + text.parse::().unwrap() + }; + let mut commands = Vec::new(); + while !bytes.is_empty() { + let count = line(&mut bytes); + let mut args = Vec::with_capacity(count); + for _ in 0..count { + let length = line(&mut bytes); + args.push(bytes[..length].to_vec()); + bytes = &bytes[length + 2..]; + } + commands.push(args); + } + commands +} + +/// Records every packed request and answers from a script, `OK` once the script runs out. +pub struct RecordingConnection { + requests: Arc>>>, + replies: Mutex>>, +} + +impl RecordingConnection { + pub fn new(replies: impl IntoIterator>) -> Self { + Self { + requests: Arc::default(), + replies: Mutex::new(replies.into_iter().collect()), + } + } + + pub fn requests(&self) -> Arc>>> { + Arc::clone(&self.requests) + } + + fn reply(&self) -> redis::RedisResult { + self.replies + .lock() + .unwrap() + .pop_front() + .unwrap_or_else(|| Ok(redis::Value::SimpleString("OK".into()))) + } +} + +impl redis::ConnectionLike for RecordingConnection { + fn req_packed_command(&mut self, command: &[u8]) -> redis::RedisResult { + self.requests.lock().unwrap().push(command.to_vec()); + self.reply() + } + + fn req_packed_commands( + &mut self, + command: &[u8], + _offset: usize, + count: usize, + ) -> redis::RedisResult> { + self.requests.lock().unwrap().push(command.to_vec()); + (0..count).map(|_| self.reply()).collect() + } + + fn get_db(&self) -> i64 { + 0 + } + + fn check_connection(&mut self) -> bool { + true + } + + fn is_open(&self) -> bool { + true + } +} diff --git a/litellm-rust/crates/cache/Cargo.toml b/litellm-rust/crates/cache/Cargo.toml index 0c504ab727a..f18dbd9cb26 100644 --- a/litellm-rust/crates/cache/Cargo.toml +++ b/litellm-rust/crates/cache/Cargo.toml @@ -7,7 +7,7 @@ repository.workspace = true [dependencies] serde.workspace = true -serde_json.workspace = true +serde_json = { workspace = true, features = ["preserve_order"] } thiserror.workspace = true [dev-dependencies] diff --git a/litellm-rust/crates/cache/src/base_cache.rs b/litellm-rust/crates/cache/src/base_cache.rs index 5c10e7fd5c3..506d944b0e8 100644 --- a/litellm-rust/crates/cache/src/base_cache.rs +++ b/litellm-rust/crates/cache/src/base_cache.rs @@ -122,36 +122,4 @@ pub trait BaseCache: Send + Sync { ) -> impl Future> + Send { self.async_set_cache(key, value, context) } - - fn disconnect(&self) -> impl Future> + Send; - - fn test_connection(&self) -> impl Future> + Send; -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use serde_json::json; - - use super::{CacheContext, SemanticCacheContext}; - - #[test] - fn semantic_context_with_ttl_only_replaces_ttl() { - let context = SemanticCacheContext { - input: Some(json!({"input": "hello"})), - messages: Some(json!([{"role": "user", "content": "hello"}])), - metadata: Some(json!({"tenant": "team"})), - scope: Some("scope".into()), - ttl: Some(Duration::from_secs(10)), - }; - - let updated = context.with_ttl(Some(Duration::from_secs(20))); - - assert_eq!(updated.ttl, Some(Duration::from_secs(20))); - assert_eq!(updated.input, context.input); - assert_eq!(updated.messages, context.messages); - assert_eq!(updated.metadata, context.metadata); - assert_eq!(updated.scope, context.scope); - } } diff --git a/litellm-rust/crates/cache/src/cache_type.rs b/litellm-rust/crates/cache/src/cache_type.rs index f0a97c04fd5..22d8c8c7cb5 100644 --- a/litellm-rust/crates/cache/src/cache_type.rs +++ b/litellm-rust/crates/cache/src/cache_type.rs @@ -55,31 +55,3 @@ impl CacheType { .find(|cache_type| cache_type.as_python_name() == value) } } - -#[cfg(test)] -mod tests { - use super::CacheType; - - #[test] - fn every_python_cache_type_has_one_round_trip_identity() { - let names = CacheType::ALL.map(CacheType::as_python_name); - assert_eq!( - names, - [ - "local", - "redis", - "redis-semantic", - "valkey-semantic", - "s3", - "disk", - "qdrant-semantic", - "azure-blob", - "gcs", - ] - ); - assert_eq!( - names.map(CacheType::from_python_name), - CacheType::ALL.map(Some) - ); - } -} diff --git a/litellm-rust/crates/cache/src/capabilities.rs b/litellm-rust/crates/cache/src/capabilities.rs index f7307e5c7bd..ac9d8fc8764 100644 --- a/litellm-rust/crates/cache/src/capabilities.rs +++ b/litellm-rust/crates/cache/src/capabilities.rs @@ -1,6 +1,6 @@ use std::{future::Future, time::Duration}; -use crate::{BaseCache, BatchEntry, Error}; +use crate::{BaseCache, BatchEntry, CacheConnectionResult, CacheContext, Error}; #[derive(Clone, Debug, PartialEq)] pub struct IncrementOperation { @@ -9,6 +9,35 @@ pub struct IncrementOperation { pub ttl: Option, } +#[derive(Clone, Debug, PartialEq)] +pub struct PushOperation { + pub key: String, + pub values: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PopOperation { + pub key: String, + pub count: Option, +} + +/// `disconnect`, for backends whose Python class releases connections or clients. +pub trait DisconnectCache: BaseCache { + fn disconnect(&self) -> impl Future> + Send; +} + +/// `test_connection`, for backends whose Python class overrides the base `NotImplementedError`. +pub trait ConnectionCache: BaseCache { + fn test_connection(&self) -> impl Future> + Send; +} + +/// `sync_ping` and `ping`. +pub trait PingCache: BaseCache { + fn sync_ping(&self) -> Result; + + fn ping(&self) -> impl Future> + Send; +} + pub trait BatchCache: BaseCache { fn batch_get_cache( &self, @@ -45,6 +74,14 @@ pub trait BatchCache: BaseCache { } } +/// `async_set_cache_pipeline_with_ttls`: one pipeline where every entry carries its own TTL. +pub trait TtlPipelineCache: BaseCache { + fn async_set_cache_pipeline_with_ttls( + &self, + entries: Vec<(String, Self::Value, Option)>, + ) -> impl Future> + Send; +} + pub trait DeleteCache: BaseCache { fn delete_cache(&self, key: &str) -> Result<(), Error>; @@ -53,6 +90,14 @@ pub trait DeleteCache: BaseCache { } } +/// `delete_cache_keys`: one round trip that reports how many keys existed. +pub trait BulkDeleteCache: DeleteCache { + fn delete_cache_keys( + &self, + keys: Vec, + ) -> impl Future> + Send; +} + pub trait FlushCache: BaseCache { fn flush_cache(&self) -> Result<(), Error>; @@ -61,18 +106,79 @@ pub trait FlushCache: BaseCache { } } -pub trait CounterCache: BaseCache { +/// `flushall`: drops every key on the server, ignoring any namespace. +pub trait FlushAllCache: FlushCache { + fn flushall(&self) -> Result<(), Error>; +} + +/// Numeric counters. Counters are independent of `Value`, so a response-valued backend can +/// expose them, the way one Python `RedisCache` serves both. +pub trait CounterCache: BaseCache { fn increment_cache(&self, key: &str, amount: f64, context: Self::Context) -> Result; + /// `refresh_ttl` re-arms the TTL on every write instead of only when the key is new; + /// backends without expiring counters ignore it, as Python's `**kwargs` does. fn async_increment( &self, key: &str, amount: f64, context: Self::Context, + _refresh_ttl: bool, ) -> impl Future> + Send { async move { self.increment_cache(key, amount, context) } } + + /// `async_increment_pipeline`, one result per operation in order. The default increments + /// one key at a time, as the in-memory cache does. + fn async_increment_pipeline( + &self, + operations: Vec, + ) -> impl Future, Error>> + Send + where + Self::Context: Default, + { + async move { + let mut results = Vec::with_capacity(operations.len()); + for operation in operations { + let context = Self::Context::default().with_ttl(operation.ttl); + results.push( + self.async_increment(&operation.key, operation.amount, context, false) + .await?, + ); + } + Ok(results) + } + } +} + +/// `batch_get_counts` and `async_batch_get_counts`: counter values read in one round trip. +pub trait CountReadCache: CounterCache { + fn batch_get_counts(&self, keys: &[String]) -> Result>, Error>; + + fn async_batch_get_counts( + &self, + keys: Vec, + ) -> impl Future>, Error>> + Send; +} + +/// `increment_with_floor`, `async_increment_with_floor`, and `async_set_max`. +pub trait BoundedCounterCache: CounterCache { + fn increment_with_floor(&self, key: &str, amount: i64, ttl: Duration) -> Result; + + fn async_increment_with_floor( + &self, + key: &str, + amount: i64, + ttl: Duration, + ) -> impl Future> + Send; + + fn async_set_max( + &self, + key: &str, + value: f64, + ttl: Option, + ) -> impl Future> + Send; } pub trait ClaimCache: BaseCache @@ -105,6 +211,17 @@ pub trait TtlCache: BaseCache { ) -> impl Future, Error>> + Send; } +pub trait RefreshTtlCache: TtlCache { + /// `async_refresh_ttl`: re-arms an existing key without touching its value. `ttl` falls + /// back to the backend default, and the result is `false` when the key is absent or + /// neither TTL is set. + fn async_refresh_ttl( + &self, + key: &str, + ttl: Option, + ) -> impl Future> + Send; +} + pub trait SetCache: BaseCache { type SetValue: Clone + Send + Sync + 'static; type SetResult: Send + Sync + 'static; @@ -127,11 +244,30 @@ pub trait QueueCache: BaseCache { values: Vec, ) -> impl Future> + Send; + /// `async_rpush_and_trim`: pushes, then keeps only the newest `max_len` entries, atomically. + /// Returns the list length right after the push, before the trim. + fn async_rpush_and_trim( + &self, + key: &str, + values: Vec, + max_len: usize, + ) -> impl Future> + Send; + + fn async_rpush_pipeline( + &self, + operations: Vec>, + ) -> impl Future, Error>> + Send; + fn async_lpop( &self, key: &str, count: Option, ) -> impl Future> + Send; + + fn async_lpop_pipeline( + &self, + operations: Vec, + ) -> impl Future, Error>> + Send; } pub trait ScanCache: BaseCache { diff --git a/litellm-rust/crates/cache/src/dual.rs b/litellm-rust/crates/cache/src/dual.rs index d68d4b2b69f..c4bc953239d 100644 --- a/litellm-rust/crates/cache/src/dual.rs +++ b/litellm-rust/crates/cache/src/dual.rs @@ -1,8 +1,8 @@ use std::{sync::Arc, time::Duration}; use crate::{ - BaseCache, BatchCache, BatchEntry, CacheConnectionResult, CacheContext, ClaimCache, - CounterCache, DeleteCache, Error, FlushCache, + BaseCache, BatchCache, BatchEntry, BulkDeleteCache, CacheContext, ClaimCache, CounterCache, + DeleteCache, Error, FlushCache, IncrementOperation, SetCache, TtlCache, }; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -33,8 +33,12 @@ pub struct DualCache { write_policy: WritePolicy, remote_failure_policy: RemoteFailurePolicy, promotion_ttl: Option, + delete_batch_size: usize, } +/// `DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE`. +pub const DEFAULT_DELETE_BATCH_SIZE: usize = 1000; + impl DualCache { pub fn new(l1: Arc, l2: Arc) -> Self { Self { @@ -44,6 +48,14 @@ impl DualCache { write_policy: WritePolicy::default(), remote_failure_policy: RemoteFailurePolicy::default(), promotion_ttl: None, + delete_batch_size: DEFAULT_DELETE_BATCH_SIZE, + } + } + + pub fn with_delete_batch_size(self, delete_batch_size: usize) -> Self { + Self { + delete_batch_size, + ..self } } @@ -217,15 +229,6 @@ where } self.l1.async_set_cache_pipeline(entries, context).await } - - async fn disconnect(&self) -> Result<(), Error> { - self.l2.disconnect().await?; - self.l1.disconnect().await - } - - async fn test_connection(&self) -> Result { - self.l2.test_connection().await - } } impl BatchCache for DualCache @@ -320,26 +323,146 @@ where } } +impl DualCache +where + C: CacheContext, + L1: BaseCache, +{ + /// Python's `local_only=True` increment: the local tier alone, read then written back. + fn increment_local(&self, key: &str, amount: f64, context: &C) -> Result { + let value = self.l1.get_cache(key, context)?.unwrap_or(0.0) + amount; + self.l1.set_cache(key, value, context)?; + Ok(value) + } +} + impl CounterCache for DualCache where C: CacheContext, L1: BaseCache, - L2: CounterCache, + L2: CounterCache, { fn increment_cache(&self, key: &str, amount: f64, context: C) -> Result { + if !self.writes_remote() { + return self.increment_local(key, amount, &context); + } let value = self.l2.increment_cache(key, amount, context.clone())?; self.l1.set_cache(key, value, &context)?; Ok(value) } - async fn async_increment(&self, key: &str, amount: f64, context: C) -> Result { + async fn async_increment( + &self, + key: &str, + amount: f64, + context: C, + refresh_ttl: bool, + ) -> Result { + if !self.writes_remote() { + return self.increment_local(key, amount, &context); + } let value = self .l2 - .async_increment(key, amount, context.clone()) + .async_increment(key, amount, context.clone(), refresh_ttl) .await?; self.l1.async_set_cache(key, value, context).await?; Ok(value) } + + /// `async_increment_cache_pipeline`, L2-first like single increments: the local tier takes + /// each remote result. + async fn async_increment_pipeline( + &self, + operations: Vec, + ) -> Result, Error> + where + C: Default, + { + if !self.writes_remote() { + return operations + .iter() + .map(|operation| { + let context = C::default().with_ttl(operation.ttl); + self.increment_local(&operation.key, operation.amount, &context) + }) + .collect(); + } + let values = self.l2.async_increment_pipeline(operations.clone()).await?; + if values.len() != operations.len() { + return Err(Error::Unavailable); + } + for (operation, value) in operations.iter().zip(&values) { + self.l1 + .async_set_cache(&operation.key, *value, C::default().with_ttl(operation.ttl)) + .await?; + } + Ok(values) + } +} + +/// `async_set_cache_sadd`: local set first, then the remote one unless writes stay local. +impl SetCache for DualCache +where + V: Clone + Send + Sync + 'static, + C: CacheContext, + S: Clone + Send + Sync + 'static, + L1: SetCache, + L2: SetCache, +{ + type SetValue = S; + type SetResult = (); + + async fn async_set_cache_sadd( + &self, + key: &str, + values: Vec, + ttl: Option, + ) -> Result<(), Error> { + self.l1 + .async_set_cache_sadd(key, values.clone(), ttl) + .await?; + if self.writes_remote() { + self.l2.async_set_cache_sadd(key, values, ttl).await?; + } + Ok(()) + } +} + +/// `async_delete_cache_keys`: every key leaves the local tier, then the remote tier in chunks +/// of `DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE`, since Redis takes a chunk as one command. +impl BulkDeleteCache for DualCache +where + V: Clone + Send + Sync + 'static, + C: CacheContext, + L1: DeleteCache, + L2: BulkDeleteCache, +{ + async fn delete_cache_keys(&self, keys: Vec) -> Result { + for key in &keys { + self.l1.delete_cache(key)?; + } + let mut deleted = 0; + for chunk in keys.chunks(self.delete_batch_size.max(1)) { + deleted += self.l2.delete_cache_keys(chunk.to_vec()).await?; + } + Ok(deleted) + } +} + +/// `async_get_ttl`: the local TTL, or the remote one when the local tier has none. +impl TtlCache for DualCache +where + V: Clone + Send + Sync + 'static, + C: CacheContext, + L1: TtlCache, + L2: TtlCache, +{ + async fn async_get_ttl(&self, key: &str) -> Result, Error> { + match self.l1.async_get_ttl(key).await? { + Some(ttl) => Ok(Some(ttl)), + None => self.l2.async_get_ttl(key).await, + } + } } impl ClaimCache for DualCache diff --git a/litellm-rust/crates/cache/src/lib.rs b/litellm-rust/crates/cache/src/lib.rs index 8364c635e3a..55d8a5fa28b 100644 --- a/litellm-rust/crates/cache/src/lib.rs +++ b/litellm-rust/crates/cache/src/lib.rs @@ -5,6 +5,7 @@ mod capabilities; mod codec; mod dual; mod error; +pub mod semantic; pub use base_cache::{ BaseCache, BatchEntry, CacheConnectionResult, CacheConnectionStatus, CacheContext, @@ -13,9 +14,13 @@ pub use base_cache::{ pub use cache_type::CacheType; pub use caching::{Cache, CacheBackend, get_cache, set_cache}; pub use capabilities::{ - BatchCache, CacheScript, ClaimCache, ClientInfoCache, CounterCache, DeleteCache, FlushCache, - IncrementOperation, QueueCache, ScanCache, ScriptCache, SetCache, TtlCache, + BatchCache, BoundedCounterCache, BulkDeleteCache, CacheScript, ClaimCache, ClientInfoCache, + ConnectionCache, CountReadCache, CounterCache, DeleteCache, DisconnectCache, FlushAllCache, + FlushCache, IncrementOperation, PingCache, PopOperation, PushOperation, QueueCache, + RefreshTtlCache, ScanCache, ScriptCache, SetCache, TtlCache, TtlPipelineCache, }; pub use codec::{CacheCodec, JsonCodec}; -pub use dual::{DualCache, ReadPolicy, RemoteFailurePolicy, WritePolicy}; +pub use dual::{ + DEFAULT_DELETE_BATCH_SIZE, DualCache, ReadPolicy, RemoteFailurePolicy, WritePolicy, +}; pub use error::Error; diff --git a/litellm-rust/crates/cache/src/semantic.rs b/litellm-rust/crates/cache/src/semantic.rs new file mode 100644 index 00000000000..c88706a213b --- /dev/null +++ b/litellm-rust/crates/cache/src/semantic.rs @@ -0,0 +1,274 @@ +//! The embedding and prompt contract every semantic backend shares. +//! +//! Python's semantic caches all read their prompt through `get_str_from_messages`, and +//! `RedisSemanticCache._get_prompt_from_kwargs` (inherited by Valkey) adds Responses API +//! `input`. Qdrant reads messages only. Each backend picks one of the two extractors here. + +use std::{future::Future, io}; + +use serde::Serialize; +use serde_json::{ + Value, + ser::{CharEscape, Formatter, Serializer}, +}; + +use crate::{BaseCache, Error, SemanticCacheContext}; + +/// Turns a prompt into the vector a semantic backend stores and searches with. +/// +/// `metadata` is the request metadata, which a host embedder may route on. Hosts that can only +/// embed asynchronously keep the default `embed`; backends that serve sync calls through a +/// runtime then block on `async_embed` instead. +pub trait Embedder: Send + Sync + 'static { + fn embed(&self, _prompt: &str, _metadata: Option<&Value>) -> Result, Error> { + Err(Error::UnsupportedOperation) + } + + fn async_embed( + &self, + prompt: &str, + metadata: Option<&Value>, + ) -> impl Future, Error>> + Send; +} + +/// One semantic read: the cached value, if any, and the similarity Python's backend writes to +/// `metadata["semantic-similarity"]`. `similarity` is `None` when the backend reports none. +#[derive(Clone, Debug, PartialEq)] +pub struct SemanticLookup { + pub value: Option, + pub similarity: Option, +} + +impl SemanticLookup { + /// A read that found nothing, with the similarity Python records for it. + pub fn miss(similarity: Option) -> Self { + Self { + value: None, + similarity, + } + } +} + +/// A semantic backend's read that also reports the similarity of the closest cached prompt, +/// the value Python stamps onto the request metadata as `semantic-similarity`. +pub trait SemanticCache: BaseCache { + fn get_cache_with_similarity( + &self, + key: &str, + context: &Self::Context, + ) -> Result, Error>; + + fn async_get_cache_with_similarity( + &self, + key: &str, + context: &Self::Context, + ) -> impl Future, Error>> + Send; +} + +/// An embedding computed ahead of time, for callers that already hold the vector. +#[derive(Clone, Debug, PartialEq)] +pub struct PreparedEmbedding(pub Vec); + +impl Embedder for PreparedEmbedding { + fn embed(&self, _prompt: &str, _metadata: Option<&Value>) -> Result, Error> { + Ok(self.0.clone()) + } + + async fn async_embed( + &self, + _prompt: &str, + _metadata: Option<&Value>, + ) -> Result, Error> { + Ok(self.0.clone()) + } +} + +/// `get_str_from_messages`: every message's text content followed by its search results. +pub fn str_from_messages(messages: &[Value]) -> String { + let mut text = String::new(); + for message in messages.iter().filter_map(Value::as_object) { + match message.get("content") { + Some(Value::String(content)) => text.push_str(content), + Some(Value::Array(parts)) => { + for part in parts { + if let Some(part_text) = part.get("text").and_then(Value::as_str) { + text.push_str(part_text); + } + } + } + _ => {} + } + push_search_results_text(&mut text, message.get("search_results")); + } + text +} + +/// The messages prompt Qdrant embeds: `None` when the request carries no messages. +pub fn prompt_from_messages(context: &SemanticCacheContext) -> Option { + let messages = context.messages.as_ref()?.as_array()?; + (!messages.is_empty()).then(|| str_from_messages(messages)) +} + +/// `RedisSemanticCache._get_prompt_from_kwargs`: chat messages first, then the text parts of a +/// Responses API `input`. `None` when neither yields a prompt. +pub fn prompt_from_context(context: &SemanticCacheContext) -> Option { + if let Some(messages) = context.messages.as_ref().and_then(Value::as_array) + && !messages.is_empty() + { + return Some(str_from_messages(messages)); + } + let input = context.input.as_ref()?; + let mut parts = Vec::new(); + collect_input_text(input, &mut parts); + let prompt = python_strip(&parts.join("\n")).to_owned(); + (!prompt.is_empty()).then_some(prompt) +} + +/// `extract_search_results_text`. +fn push_search_results_text(text: &mut String, search_results: Option<&Value>) { + let Some(Value::Array(results)) = search_results else { + return; + }; + for result in results.iter().filter_map(Value::as_object) { + for key in ["source", "title"] { + if let Some(value) = result.get(key).and_then(Value::as_str) { + text.push_str(value); + } + } + if let Some(Value::Array(content)) = result.get("content") { + for block in content.iter().filter_map(Value::as_object) { + if let Some(value) = block.get("text").and_then(Value::as_str) { + text.push_str(value); + } + } + } + if let Some(citations) = result.get("citations").filter(|value| !value.is_null()) { + text.push_str(&compact_json(citations)); + } + } +} + +fn collect_input_text(value: &Value, parts: &mut Vec) { + match value { + Value::String(text) => { + push_trimmed(text, parts); + } + Value::Array(items) => { + for item in items { + collect_input_text(item, parts); + } + } + Value::Object(map) => { + if let Some(content) = map.get("content").filter(|content| !content.is_null()) { + collect_input_text(content, parts); + return; + } + for key in ["text", "output", "input_text", "output_text"] { + if let Some(Value::String(text)) = map.get(key) + && push_trimmed(text, parts) + { + return; + } + } + } + _ => {} + } +} + +/// Pushes `text` stripped as Python's `str.strip` does, reporting whether anything was left. +fn push_trimmed(text: &str, parts: &mut Vec) -> bool { + let trimmed = python_strip(text); + if trimmed.is_empty() { + return false; + } + parts.push(trimmed.to_owned()); + true +} + +/// `str.strip()`: Python's whitespace also covers the ASCII information separators. +fn python_strip(text: &str) -> &str { + text.trim_matches(|character: char| { + character.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&character) + }) +} + +/// `json.dumps(value, separators=(",", ":"))`: compact, key insertion order, `ensure_ascii`. +fn compact_json(value: &Value) -> String { + let mut output = Vec::new(); + // Serializing a `Value` into memory cannot fail. + let _ = value.serialize(&mut Serializer::with_formatter(&mut output, AsciiFormatter)); + String::from_utf8(output).unwrap_or_default() +} + +struct AsciiFormatter; + +impl Formatter for AsciiFormatter { + fn write_string_fragment(&mut self, writer: &mut W, fragment: &str) -> io::Result<()> + where + W: ?Sized + io::Write, + { + let mut start = 0; + for (index, character) in fragment.char_indices() { + if character.is_ascii() && character != '\u{7f}' { + continue; + } + writer.write_all(&fragment.as_bytes()[start..index])?; + let mut units = [0; 2]; + for unit in character.encode_utf16(&mut units) { + write!(writer, "\\u{unit:04x}")?; + } + start = index + character.len_utf8(); + } + writer.write_all(&fragment.as_bytes()[start..]) + } + + fn write_f64(&mut self, writer: &mut W, value: f64) -> io::Result<()> + where + W: ?Sized + io::Write, + { + writer.write_all(python_float_repr(value).as_bytes()) + } + + fn write_char_escape(&mut self, writer: &mut W, escape: CharEscape) -> io::Result<()> + where + W: ?Sized + io::Write, + { + match escape { + CharEscape::AsciiControl(byte) => write!(writer, "\\u{byte:04x}"), + escape => serde_json::ser::CompactFormatter.write_char_escape(writer, escape), + } + } +} + +/// `repr(float)`: the shortest round-trip digits, positional between `1e-4` and `1e16`, and +/// otherwise scientific with a signed exponent of at least two digits. +fn python_float_repr(value: f64) -> String { + // `{:e}` yields the shortest round-trip digits, e.g. `1.5e-7`. + let scientific = format!("{value:e}"); + let (mantissa, exponent) = scientific.split_once('e').unwrap_or((&scientific, "0")); + let exponent: i32 = exponent.parse().unwrap_or(0); + let (sign, mantissa) = mantissa + .strip_prefix('-') + .map_or(("", mantissa), |rest| ("-", rest)); + let digits = mantissa.replace('.', ""); + if !(-4..16).contains(&exponent) { + let fraction = &digits[1..]; + let mantissa = if fraction.is_empty() { + digits[..1].to_owned() + } else { + format!("{}.{fraction}", &digits[..1]) + }; + let exponent_sign = if exponent < 0 { '-' } else { '+' }; + return format!("{sign}{mantissa}e{exponent_sign}{:02}", exponent.abs()); + } + let point = exponent + 1; + let positional = if point <= 0 { + format!("0.{}{digits}", "0".repeat(point.unsigned_abs() as usize)) + } else if point as usize >= digits.len() { + format!("{digits}{}.0", "0".repeat(point as usize - digits.len())) + } else { + let (whole, fraction) = digits.split_at(point as usize); + format!("{whole}.{fraction}") + }; + format!("{sign}{positional}") +} diff --git a/litellm-rust/crates/cache/tests/cache_type.rs b/litellm-rust/crates/cache/tests/cache_type.rs new file mode 100644 index 00000000000..24aaba8e5fd --- /dev/null +++ b/litellm-rust/crates/cache/tests/cache_type.rs @@ -0,0 +1,56 @@ +use litellm_cache::CacheType; +use rstest::rstest; + +#[rstest] +#[case(CacheType::Local, "local")] +#[case(CacheType::Redis, "redis")] +#[case(CacheType::RedisSemantic, "redis-semantic")] +#[case(CacheType::ValkeySemantic, "valkey-semantic")] +#[case(CacheType::S3, "s3")] +#[case(CacheType::Disk, "disk")] +#[case(CacheType::QdrantSemantic, "qdrant-semantic")] +#[case(CacheType::AzureBlob, "azure-blob")] +#[case(CacheType::Gcs, "gcs")] +fn every_python_cache_type_has_one_round_trip_identity( + #[case] cache_type: CacheType, + #[case] name: &str, +) { + assert_eq!(cache_type.as_python_name(), name); + assert_eq!(CacheType::from_python_name(name), Some(cache_type)); + assert_eq!( + serde_json::to_value(cache_type).unwrap(), + serde_json::Value::from(name) + ); + assert_eq!( + CacheType::ALL + .iter() + .filter(|candidate| candidate.as_python_name() == name) + .count(), + 1 + ); +} + +#[rstest] +fn python_cache_types_are_listed_in_python_order() { + assert_eq!( + CacheType::ALL.map(CacheType::as_python_name), + [ + "local", + "redis", + "redis-semantic", + "valkey-semantic", + "s3", + "disk", + "qdrant-semantic", + "azure-blob", + "gcs", + ] + ); +} + +#[rstest] +#[case::unknown("memcached")] +#[case::case_sensitive("Redis")] +fn unknown_python_names_have_no_cache_type(#[case] name: &str) { + assert_eq!(CacheType::from_python_name(name), None); +} diff --git a/litellm-rust/crates/cache/tests/caching.rs b/litellm-rust/crates/cache/tests/caching.rs index 36307ac9b33..baf968f4659 100644 --- a/litellm-rust/crates/cache/tests/caching.rs +++ b/litellm-rust/crates/cache/tests/caching.rs @@ -1,15 +1,25 @@ use std::{sync::Mutex, time::Duration}; use litellm_cache::{ - BaseCache, CacheConnectionResult, CacheContext, Error, ExactCacheContext, SemanticCacheContext, - get_cache, + BaseCache, CacheContext, CounterCache, Error, ExactCacheContext, IncrementOperation, + SemanticCacheContext, get_cache, }; +use rstest::{fixture, rstest}; +use serde_json::json; struct TestCache { default_ttl: Duration, writes: Mutex>, } +#[fixture] +fn cache() -> TestCache { + TestCache { + default_ttl: Duration::from_secs(60), + writes: Mutex::default(), + } +} + #[derive(Clone)] struct SemanticContext { ttl: Option, @@ -46,14 +56,6 @@ impl BaseCache for SemanticCache { fn get_cache(&self, _: &str, context: &Self::Context) -> Result, Error> { Ok((context.query == "matching prompt").then(|| "semantic hit".into())) } - - async fn disconnect(&self) -> Result<(), Error> { - Ok(()) - } - - async fn test_connection(&self) -> Result { - unreachable!() - } } impl BaseCache for TestCache { @@ -87,70 +89,124 @@ impl BaseCache for TestCache { fn get_cache(&self, _: &str, _: &ExactCacheContext) -> Result, Error> { Ok(None) } +} - async fn disconnect(&self) -> Result<(), Error> { +/// Records every `async_increment` so the default pipeline's calls are observable. +#[derive(Default)] +struct RecordingCounter { + increments: Mutex>, + total: Mutex, +} + +impl BaseCache for RecordingCounter { + type Value = f64; + type Context = ExactCacheContext; + + fn get_ttl(&self, context: &Self::Context) -> Option { + context.ttl + } + + fn set_cache(&self, _: &str, _: f64, _: &ExactCacheContext) -> Result<(), Error> { Ok(()) } - async fn test_connection(&self) -> Result { - unreachable!() + fn get_cache(&self, _: &str, _: &ExactCacheContext) -> Result, Error> { + Ok(None) } } -#[test] -fn ttl_uses_default_and_allows_per_call_override() { - let cache = TestCache { - default_ttl: Duration::from_secs(60), - writes: Mutex::default(), - }; - assert_eq!( - cache.get_ttl(&ExactCacheContext::default()), - Some(Duration::from_secs(60)) - ); +impl CounterCache for RecordingCounter { + fn increment_cache(&self, key: &str, amount: f64, _: ExactCacheContext) -> Result { + if key == "unavailable" { + return Err(Error::Unavailable); + } + let mut total = self.total.lock().unwrap(); + *total += amount; + Ok(*total) + } + + async fn async_increment( + &self, + key: &str, + amount: f64, + context: ExactCacheContext, + refresh_ttl: bool, + ) -> Result { + self.increments + .lock() + .unwrap() + .push((key.into(), amount, context.clone(), refresh_ttl)); + self.increment_cache(key, amount, context) + } +} + +fn operation(key: &str, amount: f64, ttl: Option) -> IncrementOperation { + IncrementOperation { + key: key.into(), + amount, + ttl: ttl.map(Duration::from_secs), + } +} + +#[rstest] +#[case::default_ttl(None, Some(60))] +#[case::per_call_override(Some(5), Some(5))] +fn ttl_uses_default_and_allows_per_call_override( + cache: TestCache, + #[case] ttl: Option, + #[case] expected: Option, +) { assert_eq!( cache.get_ttl(&ExactCacheContext { - ttl: Some(Duration::from_secs(5)), + ttl: ttl.map(Duration::from_secs), }), - Some(Duration::from_secs(5)) + expected.map(Duration::from_secs) ); } -#[test] -fn associated_context_preserves_backend_specific_lookup_inputs() { +#[rstest] +#[case::matching("matching prompt", Some("semantic hit"))] +#[case::other("other prompt", None)] +fn associated_context_preserves_backend_specific_lookup_inputs( + #[case] query: &str, + #[case] expected: Option<&str>, +) { let context = SemanticContext { ttl: None, - query: "matching prompt".into(), + query: query.into(), }; assert_eq!( get_cache(&SemanticCache, "shared-key", &context).unwrap(), - Some("semantic hit".into()) + expected.map(String::from) ); } -#[test] -fn semantic_context_with_ttl_preserves_lookup_inputs() { +#[rstest] +#[case::set(None, Some(30))] +#[case::replaced(Some(10), Some(20))] +#[case::cleared(Some(10), None)] +fn semantic_context_with_ttl_only_replaces_ttl( + #[case] initial: Option, + #[case] updated: Option, +) { let context = SemanticCacheContext { - input: Some(serde_json::json!("text")), - messages: Some(serde_json::json!([{"role": "user", "content": "hi"}])), - metadata: Some(serde_json::json!({"key": "value"})), + input: Some(json!({"input": "hello"})), + messages: Some(json!([{"role": "user", "content": "hello"}])), + metadata: Some(json!({"tenant": "team"})), scope: Some("scope".into()), - ttl: None, + ttl: initial.map(Duration::from_secs), }; - let updated = context.with_ttl(Some(Duration::from_secs(30))); - assert_eq!(updated.ttl(), Some(Duration::from_secs(30))); - assert_eq!(updated.input, context.input); - assert_eq!(updated.messages, context.messages); - assert_eq!(updated.metadata, context.metadata); - assert_eq!(updated.scope, context.scope); - assert_eq!(context.with_ttl(None).ttl(), None); + let result = context.with_ttl(updated.map(Duration::from_secs)); + assert_eq!(result.ttl(), updated.map(Duration::from_secs)); + assert_eq!(result.input, context.input); + assert_eq!(result.messages, context.messages); + assert_eq!(result.metadata, context.metadata); + assert_eq!(result.scope, context.scope); } +#[rstest] #[tokio::test] -async fn default_batch_operations_use_async_writes_and_stop_on_failure() { - let cache = TestCache { - default_ttl: Duration::from_secs(60), - writes: Mutex::default(), - }; +async fn default_batch_operations_use_async_writes_and_stop_on_failure(cache: TestCache) { let entry = String::from("cached"); let context = ExactCacheContext { ttl: Some(Duration::from_secs(5)), @@ -180,3 +236,101 @@ async fn default_batch_operations_use_async_writes_and_stop_on_failure() { ] ); } + +#[rstest] +#[tokio::test] +async fn default_async_increment_delegates_to_the_sync_increment() { + struct SyncOnly; + + impl BaseCache for SyncOnly { + type Value = f64; + type Context = ExactCacheContext; + + fn get_ttl(&self, context: &Self::Context) -> Option { + context.ttl + } + + fn set_cache(&self, _: &str, _: f64, _: &ExactCacheContext) -> Result<(), Error> { + Ok(()) + } + + fn get_cache(&self, _: &str, _: &ExactCacheContext) -> Result, Error> { + Ok(None) + } + } + + impl CounterCache for SyncOnly { + fn increment_cache( + &self, + _: &str, + amount: f64, + _: ExactCacheContext, + ) -> Result { + Ok(amount * 10.0) + } + } + + for refresh_ttl in [false, true] { + assert_eq!( + SyncOnly + .async_increment("key", 2.0, ExactCacheContext::default(), refresh_ttl) + .await, + Ok(20.0) + ); + } +} + +#[rstest] +#[case::empty(Vec::new(), Vec::new())] +#[case::one(vec![operation("a", 1.0, Some(10))], vec![1.0])] +#[case::in_order( + vec![operation("a", 1.0, Some(10)), operation("b", 2.5, None), operation("a", -0.5, Some(20))], + vec![1.0, 3.5, 3.0], +)] +#[tokio::test] +async fn default_increment_pipeline_increments_each_operation_in_order( + #[case] operations: Vec, + #[case] expected: Vec, +) { + let cache = RecordingCounter::default(); + assert_eq!( + cache.async_increment_pipeline(operations.clone()).await, + Ok(expected) + ); + assert_eq!( + *cache.increments.lock().unwrap(), + operations + .into_iter() + .map(|operation| ( + operation.key, + operation.amount, + ExactCacheContext { ttl: operation.ttl }, + false, + )) + .collect::>() + ); +} + +#[rstest] +#[tokio::test] +async fn default_increment_pipeline_stops_at_the_first_failure() { + let cache = RecordingCounter::default(); + assert_eq!( + cache + .async_increment_pipeline(vec![ + operation("a", 1.0, None), + operation("unavailable", 1.0, None), + operation("skipped", 1.0, None), + ]) + .await, + Err(Error::Unavailable) + ); + let keys = cache + .increments + .lock() + .unwrap() + .iter() + .map(|(key, ..)| key.clone()) + .collect::>(); + assert_eq!(keys, ["a", "unavailable"]); +} diff --git a/litellm-rust/crates/cache/tests/codec.rs b/litellm-rust/crates/cache/tests/codec.rs index e24545caad6..5320545fd49 100644 --- a/litellm-rust/crates/cache/tests/codec.rs +++ b/litellm-rust/crates/cache/tests/codec.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use litellm_cache::{CacheCodec, Error, JsonCodec}; +use rstest::rstest; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -10,7 +11,7 @@ struct RoutingState { cooldown_seconds: u64, } -#[test] +#[rstest] fn json_codec_round_trips_typed_domain_values() { let codec = JsonCodec::::new(); let value = RoutingState { @@ -25,15 +26,15 @@ fn json_codec_round_trips_typed_domain_values() { ); } -#[test] -fn json_codec_rejects_malformed_and_wrongly_typed_entries() { +#[rstest] +#[case::malformed(b"not json")] +#[case::wrongly_typed(br#"{"deployment":12}"#)] +fn json_codec_rejects_malformed_and_wrongly_typed_entries(#[case] bytes: &[u8]) { let codec = JsonCodec::::new(); - for bytes in [b"not json".as_slice(), br#"{"deployment":12}"#.as_slice()] { - assert_eq!(codec.decode(bytes).unwrap_err(), Error::InvalidEntry); - } + assert_eq!(codec.decode(bytes).unwrap_err(), Error::InvalidEntry); } -#[test] +#[rstest] fn json_codec_propagates_encoding_errors() { let codec = JsonCodec::>::new(); let value = BTreeMap::from([((1, 2), "invalid JSON object key".into())]); diff --git a/litellm-rust/crates/cache/tests/dual.rs b/litellm-rust/crates/cache/tests/dual.rs index e7e8927f8d0..87dd76388ef 100644 --- a/litellm-rust/crates/cache/tests/dual.rs +++ b/litellm-rust/crates/cache/tests/dual.rs @@ -1,12 +1,15 @@ use std::{ + collections::HashMap, sync::{Arc, Mutex}, time::Duration, }; use litellm_cache::{ - BaseCache, BatchCache, CacheConnectionResult, ClaimCache, CounterCache, DeleteCache, DualCache, - Error, ExactCacheContext, FlushCache, ReadPolicy, RemoteFailurePolicy, WritePolicy, + BaseCache, BatchCache, BatchEntry, BulkDeleteCache, ClaimCache, CounterCache, DeleteCache, + DualCache, Error, ExactCacheContext, FlushCache, IncrementOperation, ReadPolicy, + RemoteFailurePolicy, SetCache, TtlCache, WritePolicy, }; +use rstest::{fixture, rstest}; struct TestCache { value: Mutex>, @@ -41,14 +44,6 @@ where fn get_cache(&self, _: &str, _: &ExactCacheContext) -> Result, Error> { Ok(self.value.lock().unwrap().clone()) } - - async fn disconnect(&self) -> Result<(), Error> { - Ok(()) - } - - async fn test_connection(&self) -> Result { - unreachable!() - } } impl BatchCache for TestCache where V: Clone + Send + Sync + 'static {} @@ -111,7 +106,7 @@ where } } -#[test] +#[rstest] fn failed_l2_increment_leaves_l1_unchanged() { let l1 = Arc::new(TestCache::new(Some(10.0), false)); let cache = DualCache::new(l1.clone(), Arc::new(TestCache::new(Some(20.0), true))); @@ -127,7 +122,7 @@ fn failed_l2_increment_leaves_l1_unchanged() { ); } -#[test] +#[rstest] fn claim_uses_l1_fallback_without_overwriting_an_eligible_winner() { let l1 = Arc::new(TestCache::new(Some("first".to_string()), false)); let cache = DualCache::new(l1, Arc::new(TestCache::new(None, true))) @@ -193,14 +188,6 @@ impl BaseCache for SyncPanics { } Ok(()) } - - async fn disconnect(&self) -> Result<(), Error> { - Ok(()) - } - - async fn test_connection(&self) -> Result { - unreachable!() - } } impl BatchCache for SyncPanics { @@ -208,11 +195,11 @@ impl BatchCache for SyncPanics { &self, keys: Vec, context: ExactCacheContext, - ) -> Result>, Error> { + ) -> Result>, Error> { assert_eq!(keys, ["missing"]); Ok(vec![match self.0.get_cache("missing", &context)? { - Some(value) => litellm_cache::BatchEntry::Hit(value), - None => litellm_cache::BatchEntry::Miss, + Some(value) => BatchEntry::Hit(value), + None => BatchEntry::Miss, }]) } } @@ -233,6 +220,7 @@ impl FlushCache for SyncPanics { } } +#[rstest] #[tokio::test] async fn async_operations_use_the_async_l2_methods() { let l1 = Arc::new(TestCache::new(None, false)); @@ -260,7 +248,7 @@ async fn async_operations_use_the_async_l2_methods() { .async_batch_get_cache(vec!["missing".into()], context.clone()) .await .unwrap(), - [litellm_cache::BatchEntry::Hit("remote".to_string())] + [BatchEntry::Hit("remote".to_string())] ); cache .async_set_cache("missing", "written".into(), context.clone()) @@ -294,14 +282,6 @@ impl BaseCache for Unavailable { fn get_cache(&self, _: &str, _: &ExactCacheContext) -> Result, Error> { Err(Error::Unavailable) } - - async fn disconnect(&self) -> Result<(), Error> { - Ok(()) - } - - async fn test_connection(&self) -> Result { - unreachable!() - } } impl BatchCache for Unavailable {} @@ -330,7 +310,7 @@ impl ClaimCache for Unavailable { } } -#[test] +#[rstest] fn remote_failure_policy_selects_propagation_or_the_local_tier() { let context = ExactCacheContext::default(); let strict = DualCache::new(Arc::new(TestCache::new(None, false)), Arc::new(Unavailable)); @@ -353,7 +333,7 @@ fn remote_failure_policy_selects_propagation_or_the_local_tier() { assert_eq!(l1.get_cache("key", &context), Ok(None)); } -#[test] +#[rstest] fn claim_fallback_does_not_hide_non_availability_errors() { let cache = DualCache::new( Arc::new(TestCache::new(Some("first".to_string()), false)), @@ -371,7 +351,7 @@ fn claim_fallback_does_not_hide_non_availability_errors() { ); } -#[test] +#[rstest] fn local_only_policies_never_touch_l2() { let l2 = Arc::new(TestCache::new(Some("remote".to_string()), false)); let cache = DualCache::new(Arc::new(TestCache::new(None, false)), l2.clone()) @@ -383,3 +363,474 @@ fn local_only_policies_never_touch_l2() { cache.set_cache("key", "local".into(), &context).unwrap(); assert_eq!(l2.get_cache("key", &context), Ok(Some("remote".into()))); } + +type Log = Arc>>; +type StoredSet = (Vec, Option); + +/// A keyed tier that logs every call, so tests can assert which tier ran and in what order. +struct Tier { + name: &'static str, + log: Log, + fail: bool, + counters: Mutex>, + sets: Mutex>, + ttls: HashMap, +} + +impl Tier { + fn new(name: &'static str, log: &Log) -> Self { + Self { + name, + log: log.clone(), + fail: false, + counters: Mutex::default(), + sets: Mutex::default(), + ttls: HashMap::new(), + } + } + + fn failing(self) -> Self { + Self { fail: true, ..self } + } + + fn with_counter(self, key: &str, value: f64) -> Self { + self.counters + .lock() + .unwrap() + .insert(key.into(), (value, ExactCacheContext::default())); + self + } + + fn with_ttl(mut self, key: &str, seconds: u64) -> Self { + self.ttls.insert(key.into(), Duration::from_secs(seconds)); + self + } + + fn record(&self, event: String) { + self.log + .lock() + .unwrap() + .push(format!("{} {event}", self.name)); + } + + fn counter(&self, key: &str) -> Option<(f64, ExactCacheContext)> { + self.counters.lock().unwrap().get(key).cloned() + } + + fn check(&self) -> Result<(), Error> { + if self.fail { + return Err(Error::Unavailable); + } + Ok(()) + } +} + +impl BaseCache for Tier { + type Value = f64; + type Context = ExactCacheContext; + + fn get_ttl(&self, context: &Self::Context) -> Option { + context.ttl + } + + fn set_cache(&self, key: &str, value: f64, context: &ExactCacheContext) -> Result<(), Error> { + self.check()?; + self.record(format!("set {key}={value}")); + self.counters + .lock() + .unwrap() + .insert(key.into(), (value, context.clone())); + Ok(()) + } + + fn get_cache(&self, key: &str, _: &ExactCacheContext) -> Result, Error> { + Ok(self.counter(key).map(|(value, _)| value)) + } +} + +impl CounterCache for Tier { + fn increment_cache( + &self, + key: &str, + amount: f64, + context: ExactCacheContext, + ) -> Result { + self.check()?; + let mut counters = self.counters.lock().unwrap(); + let value = counters.get(key).map_or(0.0, |(value, _)| *value) + amount; + counters.insert(key.into(), (value, context)); + Ok(value) + } + + async fn async_increment( + &self, + key: &str, + amount: f64, + context: ExactCacheContext, + refresh_ttl: bool, + ) -> Result { + self.record(format!( + "increment {key}+{amount} refresh_ttl={refresh_ttl}" + )); + self.increment_cache(key, amount, context) + } + + async fn async_increment_pipeline( + &self, + operations: Vec, + ) -> Result, Error> { + let keys = operations + .iter() + .map(|operation| operation.key.as_str()) + .collect::>(); + self.record(format!("pipeline {}", keys.join(","))); + operations + .into_iter() + .map(|operation| { + self.increment_cache( + &operation.key, + operation.amount, + ExactCacheContext { ttl: operation.ttl }, + ) + }) + .collect() + } +} + +impl DeleteCache for Tier { + fn delete_cache(&self, key: &str) -> Result<(), Error> { + self.check()?; + self.record(format!("delete {key}")); + self.counters.lock().unwrap().remove(key); + Ok(()) + } +} + +impl BulkDeleteCache for Tier { + async fn delete_cache_keys(&self, keys: Vec) -> Result { + self.check()?; + self.record(format!("delete_keys {}", keys.join(","))); + let mut counters = self.counters.lock().unwrap(); + Ok(keys + .iter() + .filter(|key| counters.remove(key.as_str()).is_some()) + .count()) + } +} + +impl SetCache for Tier { + type SetValue = String; + type SetResult = (); + + async fn async_set_cache_sadd( + &self, + key: &str, + values: Vec, + ttl: Option, + ) -> Result<(), Error> { + self.check()?; + self.record(format!("sadd {key} {}", values.join(","))); + self.sets.lock().unwrap().insert(key.into(), (values, ttl)); + Ok(()) + } +} + +impl TtlCache for Tier { + async fn async_get_ttl(&self, key: &str) -> Result, Error> { + self.check()?; + self.record(format!("ttl {key}")); + Ok(self.ttls.get(key).copied()) + } +} + +#[fixture] +fn log() -> Log { + Log::default() +} + +fn events(log: &Log) -> Vec { + log.lock().unwrap().clone() +} + +fn seconds(ttl: u64) -> ExactCacheContext { + ExactCacheContext { + ttl: Some(Duration::from_secs(ttl)), + } +} + +#[rstest] +#[case::window_semantics(false)] +#[case::refresh_on_every_write(true)] +#[tokio::test] +async fn async_increment_passes_refresh_ttl_to_l2_and_stores_its_result_locally( + log: Log, + #[case] refresh_ttl: bool, +) { + let l1 = Arc::new(Tier::new("l1", &log).with_counter("counter", 1.0)); + let l2 = Arc::new(Tier::new("l2", &log).with_counter("counter", 10.0)); + let cache = DualCache::new(l1.clone(), l2.clone()); + + assert_eq!( + cache + .async_increment("counter", 2.0, seconds(30), refresh_ttl) + .await, + Ok(12.0) + ); + assert_eq!( + events(&log), + [ + format!("l2 increment counter+2 refresh_ttl={refresh_ttl}"), + "l1 set counter=12".into(), + ] + ); + assert_eq!(l1.counter("counter"), Some((12.0, seconds(30)))); + assert_eq!(l2.counter("counter"), Some((12.0, seconds(30)))); +} + +#[rstest] +#[tokio::test] +async fn failed_async_l2_increment_leaves_l1_unchanged(log: Log) { + let l1 = Arc::new(Tier::new("l1", &log).with_counter("counter", 1.0)); + let cache = DualCache::new(l1.clone(), Arc::new(Tier::new("l2", &log).failing())) + .with_remote_failure_policy(RemoteFailurePolicy::UseLocal); + + assert_eq!( + cache + .async_increment("counter", 2.0, seconds(30), true) + .await, + Err(Error::Unavailable) + ); + assert_eq!( + l1.counter("counter"), + Some((1.0, ExactCacheContext::default())) + ); +} + +#[rstest] +#[case::empty(Vec::new(), Vec::new())] +#[case::one_key(vec![("a", 1.0, Some(10))], vec![6.0])] +#[case::repeated_and_mixed_ttls( + vec![("a", 1.0, Some(10)), ("b", 2.0, None), ("a", 3.0, Some(20))], + vec![6.0, 2.0, 9.0], +)] +#[tokio::test] +async fn async_increment_pipeline_runs_l2_first_and_l1_takes_each_remote_result( + log: Log, + #[case] operations: Vec<(&str, f64, Option)>, + #[case] expected: Vec, +) { + let operations = operations + .into_iter() + .map(|(key, amount, ttl)| IncrementOperation { + key: key.into(), + amount, + ttl: ttl.map(Duration::from_secs), + }) + .collect::>(); + let l1 = Arc::new(Tier::new("l1", &log).with_counter("a", 100.0)); + let l2 = Arc::new(Tier::new("l2", &log).with_counter("a", 5.0)); + let cache = DualCache::new(l1.clone(), l2); + + assert_eq!( + cache.async_increment_pipeline(operations.clone()).await, + Ok(expected.clone()) + ); + let keys = operations + .iter() + .map(|operation| operation.key.as_str()) + .collect::>(); + let mut expected_events = vec![format!("l2 pipeline {}", keys.join(","))]; + expected_events.extend( + operations + .iter() + .zip(&expected) + .map(|(operation, value)| format!("l1 set {}={value}", operation.key)), + ); + assert_eq!(events(&log), expected_events); + if let Some((operation, value)) = operations.iter().zip(&expected).next_back() { + assert_eq!( + l1.counter(&operation.key), + Some((*value, ExactCacheContext { ttl: operation.ttl })) + ); + } +} + +#[rstest] +#[tokio::test] +async fn failed_l2_increment_pipeline_leaves_l1_unchanged(log: Log) { + let l1 = Arc::new(Tier::new("l1", &log).with_counter("a", 1.0)); + let cache = DualCache::new(l1.clone(), Arc::new(Tier::new("l2", &log).failing())); + + assert_eq!( + cache + .async_increment_pipeline(vec![IncrementOperation { + key: "a".into(), + amount: 1.0, + ttl: None, + }]) + .await, + Err(Error::Unavailable) + ); + assert_eq!(events(&log), ["l2 pipeline a"]); + assert_eq!(l1.counter("a"), Some((1.0, ExactCacheContext::default()))); +} + +#[rstest] +#[case::both_tiers(WritePolicy::Both, &["l1 sadd members a,b", "l2 sadd members a,b"])] +#[case::local_only(WritePolicy::LocalOnly, &["l1 sadd members a,b"])] +#[tokio::test] +async fn set_add_writes_locally_then_remotely_unless_local_only( + log: Log, + #[case] write_policy: WritePolicy, + #[case] expected: &[&str], +) { + let l1 = Arc::new(Tier::new("l1", &log)); + let l2 = Arc::new(Tier::new("l2", &log)); + let cache = DualCache::new(l1.clone(), l2.clone()).with_write_policy(write_policy); + let ttl = Some(Duration::from_secs(45)); + + cache + .async_set_cache_sadd("members", vec!["a".into(), "b".into()], ttl) + .await + .unwrap(); + assert_eq!(events(&log), expected); + let stored = Some((vec!["a".to_string(), "b".to_string()], ttl)); + assert_eq!(l1.sets.lock().unwrap().get("members").cloned(), stored); + assert_eq!( + l2.sets.lock().unwrap().get("members").cloned(), + stored.filter(|_| write_policy == WritePolicy::Both) + ); +} + +#[rstest] +#[tokio::test] +async fn failed_local_set_add_never_reaches_l2(log: Log) { + let cache = DualCache::new( + Arc::new(Tier::new("l1", &log).failing()), + Arc::new(Tier::new("l2", &log)), + ); + assert_eq!( + cache + .async_set_cache_sadd("members", vec!["a".into()], None) + .await, + Err(Error::Unavailable) + ); + assert!(events(&log).is_empty()); +} + +#[rstest] +#[case::empty(None, &[], &[])] +#[case::default_batch_size(None, &["a", "b", "c"], &["a,b,c"])] +#[case::chunked(Some(2), &["a", "b", "c", "d", "e"], &["a,b", "c,d", "e"])] +#[case::exact_chunks(Some(2), &["a", "b", "c", "d"], &["a,b", "c,d"])] +#[case::zero_means_one_per_chunk(Some(0), &["a", "b"], &["a", "b"])] +#[tokio::test] +async fn bulk_delete_removes_every_key_locally_then_remotely_in_chunks( + log: Log, + #[case] batch_size: Option, + #[case] keys: &[&str], + #[case] chunks: &[&str], +) { + let l1 = Tier::new("l1", &log); + let l2 = Tier::new("l2", &log); + for key in keys.iter().step_by(2) { + l2.counters + .lock() + .unwrap() + .insert((*key).into(), (1.0, ExactCacheContext::default())); + } + let l1 = Arc::new(l1); + let mut cache = DualCache::new(l1, Arc::new(l2)); + if let Some(batch_size) = batch_size { + cache = cache.with_delete_batch_size(batch_size); + } + + assert_eq!( + cache + .delete_cache_keys(keys.iter().map(|key| (*key).into()).collect()) + .await, + Ok(keys.len().div_ceil(2)) + ); + let expected = keys + .iter() + .map(|key| format!("l1 delete {key}")) + .chain(chunks.iter().map(|chunk| format!("l2 delete_keys {chunk}"))) + .collect::>(); + assert_eq!(events(&log), expected); +} + +#[rstest] +#[tokio::test] +async fn bulk_delete_stops_before_l2_when_the_local_delete_fails(log: Log) { + let cache = DualCache::new( + Arc::new(Tier::new("l1", &log).failing()), + Arc::new(Tier::new("l2", &log)), + ); + assert_eq!( + cache.delete_cache_keys(vec!["a".into()]).await, + Err(Error::Unavailable) + ); + assert!(events(&log).is_empty()); +} + +#[rstest] +#[case::local_hit("both", Some(10), &["l1 ttl both"])] +#[case::remote_fallback("remote", Some(20), &["l1 ttl remote", "l2 ttl remote"])] +#[case::missing_everywhere("missing", None, &["l1 ttl missing", "l2 ttl missing"])] +#[tokio::test] +async fn ttl_reads_local_then_remote( + log: Log, + #[case] key: &str, + #[case] expected: Option, + #[case] expected_events: &[&str], +) { + let cache = DualCache::new( + Arc::new(Tier::new("l1", &log).with_ttl("both", 10)), + Arc::new( + Tier::new("l2", &log) + .with_ttl("both", 99) + .with_ttl("remote", 20), + ), + ); + assert_eq!( + cache.async_get_ttl(key).await, + Ok(expected.map(Duration::from_secs)) + ); + assert_eq!(events(&log), expected_events); +} + +/// Python `local_only=True`: the increment and the pipeline stay on the local tier. +#[rstest] +#[tokio::test] +async fn local_only_writes_increment_the_local_tier_alone(log: Log) { + let l1 = Arc::new(Tier::new("l1", &log).with_counter("a", 1.0)); + let l2 = Arc::new(Tier::new("l2", &log).with_counter("a", 10.0)); + let cache = DualCache::new(l1.clone(), l2.clone()).with_write_policy(WritePolicy::LocalOnly); + + assert_eq!(cache.increment_cache("a", 2.0, seconds(30)), Ok(3.0)); + assert_eq!( + cache.async_increment("a", 1.0, seconds(30), true).await, + Ok(4.0) + ); + let operations = vec![ + IncrementOperation { + key: "a".into(), + amount: 1.0, + ttl: Some(Duration::from_secs(10)), + }, + IncrementOperation { + key: "b".into(), + amount: 2.0, + ttl: None, + }, + ]; + assert_eq!( + cache.async_increment_pipeline(operations).await, + Ok(vec![5.0, 2.0]) + ); + assert_eq!( + events(&log), + ["l1 set a=3", "l1 set a=4", "l1 set a=5", "l1 set b=2"] + ); + assert_eq!(l2.counter("a"), Some((10.0, ExactCacheContext::default()))); +} diff --git a/litellm-rust/crates/cache/tests/semantic.rs b/litellm-rust/crates/cache/tests/semantic.rs new file mode 100644 index 00000000000..97a552a8010 --- /dev/null +++ b/litellm-rust/crates/cache/tests/semantic.rs @@ -0,0 +1,270 @@ +use litellm_cache::{ + Error, SemanticCacheContext, + semantic::{ + Embedder, PreparedEmbedding, prompt_from_context, prompt_from_messages, str_from_messages, + }, +}; +use rstest::rstest; +use serde_json::{Value, json}; + +fn context(messages: Option, input: Option) -> SemanticCacheContext { + SemanticCacheContext { + messages, + input, + ..SemanticCacheContext::default() + } +} + +#[rstest] +#[case::empty(json!([]), "")] +#[case::string_content(json!([{"role": "user", "content": "hello"}]), "hello")] +#[case::concatenates_messages( + json!([{"role": "system", "content": "be brief. "}, {"role": "user", "content": "hello"}]), + "be brief. hello", +)] +#[case::text_parts( + json!([{"role": "user", "content": [ + {"type": "text", "text": "What is "}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, + {"type": "text", "text": "this?"}, + ]}]), + "What is this?", +)] +#[case::missing_null_and_empty_content( + json!([{"role": "assistant"}, {"role": "assistant", "content": null}, {"role": "user", "content": ""}]), + "", +)] +#[case::search_results_hidden_behind_small_content( + json!([{"role": "tool", "content": "small", "search_results": [ + {"source": "s", "title": "t", "content": [{"text": "hidden payload"}]}, + ]}]), + "smallsthidden payload", +)] +#[case::title_only_search_result( + json!([{"role": "tool", "content": "small", "search_results": [ + {"source": "s", "title": "long title", "content": []}, + ]}]), + "smallslong title", +)] +#[case::search_results_without_content( + json!([{"role": "tool", "search_results": [{"source": "s", "title": "t"}]}]), + "st", +)] +#[case::search_result_fields_in_python_order( + json!([{"role": "tool", "content": "c", "search_results": [ + {"citations": {"enabled": true}, "content": [{"text": "body"}], "title": "t", "source": "s"}, + {"source": "s2"}, + ]}]), + r#"cstbody{"enabled":true}s2"#, +)] +#[case::null_citations_skipped( + json!([{"role": "tool", "content": "c", "search_results": [ + {"source": "s", "citations": null}, + ]}]), + "cs", +)] +#[case::non_string_and_non_object_entries_skipped( + json!([{"role": "tool", "content": "c", "search_results": [ + "junk", + {"source": 1, "title": null, "content": ["junk", {"text": 3}, {"text": "kept"}]}, + ]}]), + "ckept", +)] +#[case::non_list_search_results_skipped( + json!([{"role": "tool", "content": "c", "search_results": {"source": "s"}}]), + "c", +)] +#[case::citations_compact_in_insertion_order( + json!([{"role": "tool", "search_results": [ + {"citations": {"z": 1, "a": [1.5, true, null], "m": {"k": "v"}}}, + ]}]), + r#"{"z":1,"a":[1.5,true,null],"m":{"k":"v"}}"#, +)] +#[case::citations_ensure_ascii( + json!([{"role": "tool", "search_results": [{"citations": ["caf\u{e9}", "\u{4e2d}"]}]}]), + r#"["caf\u00e9","\u4e2d"]"#, +)] +#[case::citations_astral_chars_as_surrogate_pairs( + json!([{"role": "tool", "search_results": [{"citations": "\u{1f600}"}]}]), + r#""\ud83d\ude00""#, +)] +#[case::citations_escapes( + json!([{"role": "tool", "search_results": [{"citations": "q\"\\\n\t\u{1}/"}]}]), + r#""q\"\\\n\t\u0001/""#, +)] +#[case::citations_large_float_exponent( + json!([{"role": "tool", "search_results": [{"citations": [1e20, 1.0]}]}]), + "[1e+20,1.0]", +)] +#[case::citations_scalars( + json!([{"role": "tool", "search_results": [{"citations": false}, {"citations": 3}]}]), + "false3", +)] +fn str_from_messages_matches_python(#[case] messages: Value, #[case] expected: &str) { + assert_eq!(str_from_messages(messages.as_array().unwrap()), expected); +} + +#[rstest] +#[case::no_messages(None, None)] +#[case::empty_messages(Some(json!([])), None)] +#[case::messages_not_a_list(Some(json!("hello")), None)] +#[case::messages(Some(json!([{"content": "hello"}])), Some("hello"))] +#[case::messages_without_text(Some(json!([{"content": null}])), Some(""))] +fn prompt_from_messages_reads_messages_only( + #[case] messages: Option, + #[case] expected: Option<&str>, +) { + let context = context(messages, Some(json!("responses prompt"))); + assert_eq!(prompt_from_messages(&context).as_deref(), expected); +} + +#[rstest] +#[case::prefers_messages( + Some(json!([{"content": "message prompt"}])), + Some(json!("responses prompt")), + Some("message prompt"), +)] +#[case::empty_messages_fall_back_to_input( + Some(json!([])), + Some(json!("responses prompt")), + Some("responses prompt"), +)] +#[case::messages_without_text_keep_an_empty_prompt( + Some(json!([{"content": null}])), + Some(json!("x")), + Some(""), +)] +#[case::nothing(None, None, None)] +#[case::null_input(None, Some(Value::Null), None)] +#[case::blank_string(None, Some(json!(" ")), None)] +#[case::trimmed_string( + None, + Some(json!(" What is the capital of France?\n")), + Some("What is the capital of France?"), +)] +#[case::image_only( + None, + Some(json!([{"type": "input_image", "image_url": "https://example.com"}])), + None, +)] +#[case::structured_input( + None, + Some(json!([{"role": "user", "content": [ + {"type": "input_text", "text": "What is the capital of France?"}, + {"type": "input_text", "text": "Answer briefly."}, + {"type": "input_image", "image_url": "https://example.com/paris.png"}, + ]}])), + Some("What is the capital of France?\nAnswer briefly."), +)] +#[case::model_objects_after_dump( + None, + Some(json!([ + {"content": [{"text": "model dump prompt"}]}, + {"content": [{"output_text": "dict prompt"}]}, + {"content": [{"input_text": "inline prompt"}]}, + {"content": [{"type": "input_image", "image_url": "https://example.com"}]}, + ])), + Some("model dump prompt\ndict prompt\ninline prompt"), +)] +#[case::object_content( + None, + Some(json!({"content": [{"text": "object content prompt"}]})), + Some("object content prompt"), +)] +#[case::string_content(None, Some(json!({"content": " inline "})), Some("inline"))] +#[case::null_content_uses_text_keys( + None, + Some(json!({"content": null, "output": "tool output"})), + Some("tool output"), +)] +#[case::content_wins_over_text(None, Some(json!({"content": [], "text": "ignored"})), None)] +#[case::text_key_precedence( + None, + Some(json!({"output_text": "d", "input_text": "c", "output": "b", "text": "a"})), + Some("a"), +)] +#[case::input_text_key(None, Some(json!({"input_text": "only input"})), Some("only input"))] +#[case::output_text_key(None, Some(json!({"output_text": "only output"})), Some("only output"))] +#[case::non_string_text_keys_skipped( + None, + Some(json!({"text": 1, "output": "fallback"})), + Some("fallback"), +)] +#[case::nested_lists(None, Some(json!([["a", [" b "]], "", "c"])), Some("a\nb\nc"))] +#[case::scalars_ignored(None, Some(json!([1, true, null, "kept"])), Some("kept"))] +fn prompt_from_context_matches_python( + #[case] messages: Option, + #[case] input: Option, + #[case] expected: Option<&str>, +) { + assert_eq!( + prompt_from_context(&context(messages, input)).as_deref(), + expected + ); +} + +/// Python `test_redis_semantic_cache_prompt_extraction_skips_blank_dict_text_keys`: a blank +/// text key falls through to the next one. +#[rstest] +#[case::blank_text_falls_through( + json!({"text": " ", "input_text": "fallback prompt"}), + "fallback prompt", +)] +fn prompt_from_context_skips_blank_text_keys(#[case] input: Value, #[case] expected: &str) { + assert_eq!( + prompt_from_context(&context(None, Some(input))).as_deref(), + Some(expected) + ); +} + +/// Where `json.dumps(..., separators=(",", ":"))` and Python `str.strip` differ from +/// `semantic.rs`: ensure_ascii escapes DEL, small floats keep Python's two-digit exponent, and +/// strip also removes the ASCII information separators. +#[rstest] +#[case::del_is_escaped(json!([{"search_results": [{"citations": "\u{7f}"}]}]), None, r#""\u007f""#)] +#[case::small_float_exponent(json!([{"search_results": [{"citations": 1.5e-7}]}]), None, "1.5e-07")] +#[case::float_at_positional_floor(json!([{"search_results": [{"citations": 1e-4}]}]), None, "0.0001")] +#[case::float_at_scientific_ceiling(json!([{"search_results": [{"citations": 1e16}]}]), None, "1e+16")] +#[case::large_float(json!([{"search_results": [{"citations": [1.25e20, -2.5, 3.0]}]}]), None, "[1.25e+20,-2.5,3.0]")] +#[case::strip_information_separators(json!([]), Some(json!("\u{1c}a\u{1f}")), "a")] +fn python_serialization_edge_cases( + #[case] messages: Value, + #[case] input: Option, + #[case] expected: &str, +) { + let actual = match input { + Some(input) => prompt_from_context(&context(None, Some(input))).unwrap_or_default(), + None => str_from_messages(messages.as_array().unwrap()), + }; + assert_eq!(actual, expected); +} + +#[rstest] +#[tokio::test] +async fn prepared_embedding_returns_its_vector_for_any_prompt() { + let embedding = PreparedEmbedding(vec![0.1, 0.2, 0.3]); + let metadata = json!({"tenant": "team"}); + assert_eq!(embedding.embed("a", None), Ok(vec![0.1, 0.2, 0.3])); + assert_eq!( + embedding.async_embed("b", Some(&metadata)).await, + Ok(vec![0.1, 0.2, 0.3]) + ); +} + +#[rstest] +#[tokio::test] +async fn embedders_default_to_async_only() { + struct AsyncOnly; + + impl Embedder for AsyncOnly { + async fn async_embed(&self, prompt: &str, _: Option<&Value>) -> Result, Error> { + Ok(vec![prompt.len() as f32]) + } + } + + assert_eq!( + AsyncOnly.embed("abc", None), + Err(Error::UnsupportedOperation) + ); + assert_eq!(AsyncOnly.async_embed("abc", None).await, Ok(vec![3.0])); +} diff --git a/litellm-rust/crates/core-utils/src/serde_compat.rs b/litellm-rust/crates/core-utils/src/serde_compat.rs index c767c709f50..fddab1d80e3 100644 --- a/litellm-rust/crates/core-utils/src/serde_compat.rs +++ b/litellm-rust/crates/core-utils/src/serde_compat.rs @@ -17,6 +17,11 @@ pub fn parse_str_bool(value: &str) -> Option { token.eq_ignore_ascii_case("false").then_some(false) } +/// `redis-py` string Booleans: only `1`, `true`, and `yes` (case-insensitive) are true. +pub fn parse_redis_bool(value: &str) -> bool { + value == "1" || value.eq_ignore_ascii_case("true") || value.eq_ignore_ascii_case("yes") +} + impl<'de> DeserializeAs<'de, i64> for LaxI64 { fn deserialize_as>(deserializer: D) -> Result { deserializer.deserialize_any(Self) diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 69ae8004d46..3bfc5bae925 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -36,6 +36,7 @@ url.workspace = true veil.workspace = true [dev-dependencies] +litellm-secrets.workspace = true litellm-auth-gcp.workspace = true litellm-llms = { workspace = true, features = ["test-support"] } rstest.workspace = true diff --git a/litellm-rust/crates/core/src/ocr/handler.rs b/litellm-rust/crates/core/src/ocr/handler.rs index 19037e49033..c1265e1e91c 100644 --- a/litellm-rust/crates/core/src/ocr/handler.rs +++ b/litellm-rust/crates/core/src/ocr/handler.rs @@ -22,7 +22,12 @@ pub(crate) async fn perform_ocr_request( ) -> Result { request.response_format()?; let config = request.config; - let request = prepare_request(request, caller_document, client); + let secrets = client + .secret_source() + .resolve(&config.secret_names()) + .await + .map_err(|error| Error::Secret(std::sync::Arc::new(error)))?; + let request = prepare_request(request, caller_document, client, secrets); let hooks = OcrCallHooks::new(host.clone(), &request, config); config.ocr(client, &request, &hooks).await } diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/core/src/ocr/prepare.rs index 715aedc69df..54960256faa 100644 --- a/litellm-rust/crates/core/src/ocr/prepare.rs +++ b/litellm-rust/crates/core/src/ocr/prepare.rs @@ -1,7 +1,10 @@ use litellm_auth::{InputSource, SecretValue, Sourced}; -use litellm_llms::base_llm::ocr::{ - handler::OcrClient, - transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest}, +use litellm_llms::base_llm::{ + inference::secrets::Secrets, + ocr::{ + handler::OcrClient, + transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest}, + }, }; use super::provider_config::OcrProvider; @@ -11,6 +14,7 @@ pub(crate) fn prepare_request( request: ResolvedOcrRequest, caller_document: bool, client: &OcrClient, + secrets: Secrets, ) -> PreparedOcrRequest { let credentials = request.credentials.clone(); let (preferred_api_key_env, api_base_env) = match request.config.provider() { @@ -24,7 +28,7 @@ pub(crate) fn prepare_request( | OcrProvider::Reducto | OcrProvider::VertexAi => (None, None), }; - let secret = |name: &str| client.secrets().truthy(name); + let secret = |name: &str| secrets.truthy(name); let dynamic_api_key = credentials.dynamic_api_key.or_else(|| { credentials.api_key.clone().or_else(|| { preferred_api_key_env @@ -60,12 +64,7 @@ pub(crate) fn prepare_request( PreparedOcrRequest { model, document, - connection: OcrConnection::new( - resolved, - transport, - client.settings().clone(), - client.secrets().clone(), - ), + connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets), caller_document, optional_params, input_sources, @@ -79,6 +78,7 @@ pub(crate) fn prepare_request_for_test(request: ResolvedOcrRequest) -> PreparedO request, true, &OcrClient::for_test(reqwest::Client::new(), reqwest::Client::new()), + std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment), ) } diff --git a/litellm-rust/crates/core/src/ocr/provider_config.rs b/litellm-rust/crates/core/src/ocr/provider_config.rs index d38d87b92cc..0b09e9be354 100644 --- a/litellm-rust/crates/core/src/ocr/provider_config.rs +++ b/litellm-rust/crates/core/src/ocr/provider_config.rs @@ -114,6 +114,10 @@ impl OcrConfigKind { with_config!(self, config => config.get_api_key_env_var()) } + pub(crate) fn secret_names(self) -> Vec<&'static str> { + with_config!(self, config => config.secret_names()) + } + pub(crate) fn get_health_check_document(self) -> OcrDocument { with_config!(self, config => config.get_health_check_document()) } @@ -213,6 +217,8 @@ fn is_document_intelligence_model(model: &str) -> bool { #[cfg(test)] mod tests { + use std::collections::HashSet; + use litellm_auth::{InputSource, Sourced}; use litellm_llms::{ base_llm::ocr::document::InlineDocument, cohere::ocr::transformation::validate_document, @@ -221,6 +227,27 @@ mod tests { use super::*; + #[rstest] + #[case(OcrConfigKind::AwsTextract)] + #[case(OcrConfigKind::AwsTextractAnalyze)] + #[case(OcrConfigKind::Cohere)] + #[case(OcrConfigKind::Mistral)] + #[case(OcrConfigKind::AzureAi)] + #[case(OcrConfigKind::AzureCohere)] + #[case(OcrConfigKind::AzureDocumentIntelligence)] + #[case(OcrConfigKind::ReductoLegacy)] + #[case(OcrConfigKind::ReductoV3)] + #[case(OcrConfigKind::VertexAi)] + #[case(OcrConfigKind::VertexDeepSeek)] + fn secret_names_include_api_keys_without_duplicates(#[case] config: OcrConfigKind) { + let names = config.secret_names(); + let unique = names.iter().collect::>(); + assert_eq!(names.len(), unique.len()); + if let Some(api_key) = config.get_api_key_env_var() { + assert!(names.contains(&api_key)); + } + } + #[rstest] #[case("cohere")] #[case("mistral")] diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs index 3aedc7b9023..d376f0df784 100644 --- a/litellm-rust/crates/core/tests/ocr.rs +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -1,5 +1,6 @@ use std::sync::{Arc, Mutex}; +use futures_util::future::BoxFuture; use litellm_auth_gcp::VertexAuth; use litellm_host::{ event::{CallEvent, MachineEvent, WireRequest}, @@ -10,11 +11,14 @@ use litellm_http::{ HttpClientPool, HttpSettings, Resolution, media::{PublicDnsResolver, UrlPolicy}, }; +use litellm_llms::base_llm::inference::secrets::{SecretSource, Secrets}; use litellm_llms::base_llm::ocr::{ error::Error as OcrError, handler::OcrClient, settings::OcrSettings, - transformation::{LiteLLMOcrResponse, OCR_RESPONSE_MAX_BYTES, OcrTransportConfig}, + transformation::{ + BaseOcrConfig, LiteLLMOcrResponse, OCR_RESPONSE_MAX_BYTES, OcrTransportConfig, + }, }; use rstest::rstest; use serde_json::{Value, json}; @@ -27,6 +31,32 @@ use super::{ }; use crate::ocr::route::{LocalOcrHost, OcrOp, OcrOpResult, ocr_machine}; +struct RecordingSecretSource { + names: Arc>>, + values: &'static [(&'static str, &'static str)], + api_base: String, +} + +impl SecretSource for RecordingSecretSource { + fn resolve<'a>( + &'a self, + names: &'a [&'static str], + ) -> BoxFuture<'a, Result> { + *self.names.lock().unwrap() = names.to_vec(); + let values = self.values; + let api_base = self.api_base.clone(); + Box::pin(async move { + Ok(Arc::new(move |name: &str| match name { + "MISTRAL_AZURE_API_BASE" => Some(api_base.clone()), + _ => values + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.to_string()), + }) as Secrets) + }) + } +} + #[rstest] #[case::mistral("mistral/model", json!({}))] #[case::vertex("vertex_ai/mistral-ocr-latest", json!({"vertex_project":"test-project", "vertex_location":"us-central1"}))] @@ -184,14 +214,11 @@ async fn mistral_env_fallbacks_follow_python_through_the_injected_secret_source( #[case] expected_key: &str, ) { let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; - let secret_base = base.clone(); - let client = ocr_client().with_secrets(Arc::new(move |name: &str| match name { - "MISTRAL_AZURE_API_BASE" => Some(secret_base.clone()), - "MISTRAL_API_BASE" => Some("http://127.0.0.1:9/never-read".into()), - _ => secrets - .iter() - .find(|(key, _)| *key == name) - .map(|(_, value)| value.to_string()), + let names = Arc::new(Mutex::new(Vec::new())); + let client = ocr_client().with_secrets(Arc::new(RecordingSecretSource { + names: names.clone(), + values: secrets, + api_base: base.clone(), })); let request = decode_request(OcrWireRequest { model: "mistral/model".into(), @@ -208,9 +235,47 @@ async fn mistral_env_fallbacks_follow_python_through_the_injected_secret_source( crate::ocr::client::perform(&client, request).await.unwrap(); server.await.unwrap(); + assert_eq!( + *names.lock().unwrap(), + litellm_llms::mistral::ocr::transformation::MistralOcrConfig.secret_names() + ); assert!(seen.lock().unwrap()[0].contains(&format!("authorization: Bearer {expected_key}"))); } +#[tokio::test] +async fn mistral_ocr_resolves_provider_secrets_before_transformation() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; + let names = Arc::new(Mutex::new(Vec::new())); + let client = ocr_client().with_secrets(Arc::new(RecordingSecretSource { + names: names.clone(), + values: &[("MISTRAL_API_KEY", "source-key")], + api_base: base.clone(), + })); + let request = decode_request(OcrWireRequest { + model: "mistral/mistral-ocr-latest".into(), + document: json!({ + "type":"document_url", + "document_url":"data:application/pdf;base64,YWJj" + }), + api_key: None, + api_base: None, + custom_llm_provider: None, + extra_headers: None, + optional_params: Default::default(), + input_sources: Default::default(), + timeout_seconds: Some(2.0), + }) + .unwrap(); + + crate::ocr::client::perform(&client, request).await.unwrap(); + server.await.unwrap(); + assert_eq!( + *names.lock().unwrap(), + litellm_llms::mistral::ocr::transformation::MistralOcrConfig.secret_names() + ); + assert!(seen.lock().unwrap()[0].contains("authorization: Bearer source-key")); +} + #[tokio::test] async fn ocr_client_uses_the_injected_http_pool_configuration() { let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; @@ -224,7 +289,7 @@ async fn ocr_client_uses_the_injected_http_pool_configuration() { UrlPolicy::default(), VertexAuth::default(), OcrSettings::default(), - Arc::new(litellm_core_utils::settings::ProcessEnvironment), + Arc::new(litellm_llms::base_llm::inference::secrets::EnvironmentSecrets), ) .unwrap(); crate::ocr::client::perform(&client, wire_request("mistral/model", &base, json!({}))) diff --git a/litellm-rust/crates/host-python/src/execution.rs b/litellm-rust/crates/host-python/src/execution.rs index 083c184e37e..b435bf241d2 100644 --- a/litellm-rust/crates/host-python/src/execution.rs +++ b/litellm-rust/crates/host-python/src/execution.rs @@ -29,7 +29,7 @@ pyo3::create_exception!( static FORK_GATE: ForkGate = ForkGate::new(); -/// Whether this process has started the Tokio runtime. +/// Whether this process has entered process-bound native execution. pub fn runtime_started() -> bool { FORK_GATE.started(std::process::id()) } @@ -40,9 +40,8 @@ pub fn reserve_process_for_forking() -> Result<(), RuntimeAlreadyStarted> { FORK_GATE.reserve(std::process::id()) } -/// The only door to the Tokio runtime: every route reaches it through this module, which is -/// what lets the gate speak for the whole extension. `clippy.toml` disallows going around it. -fn enter_runtime() -> PyResult<()> { +/// Claims process-bound native state before runtime startup or tokenizer execution. +pub fn enter_native() -> PyResult<()> { FORK_GATE .enter(std::process::id()) .map_err(|refused| match refused { @@ -60,7 +59,7 @@ fn enter_runtime() -> PyResult<()> { #[expect(clippy::disallowed_methods, reason = "this is the gated door")] fn runtime() -> PyResult<&'static Runtime> { - enter_runtime()?; + enter_native()?; Ok(pyo3_async_runtimes::tokio::get_runtime()) } @@ -70,7 +69,7 @@ where F: Future> + Send + 'static, T: for<'py> IntoPyObject<'py> + Send + 'static, { - enter_runtime()?; + enter_native()?; pyo3_async_runtimes::tokio::future_into_py(py, future) } diff --git a/litellm-rust/crates/host-python/src/lib.rs b/litellm-rust/crates/host-python/src/lib.rs index 7d164ab7535..4a33975a918 100644 --- a/litellm-rust/crates/host-python/src/lib.rs +++ b/litellm-rust/crates/host-python/src/lib.rs @@ -20,7 +20,7 @@ pub use argument::lookup; pub use callable::wrap_failure; pub use driver::run_call; pub use execution::{ - ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, poll_async_value, + ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, enter_native, poll_async_value, reserve_process_for_forking, run_async, run_async_value, run_sync, run_sync_value, runtime_started, }; diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml index f04b78feee1..ed15d9f7cdb 100644 --- a/litellm-rust/crates/llms/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -18,6 +18,7 @@ litellm-auth-gcp.workspace = true litellm-host.workspace = true litellm-framing.workspace = true litellm-http.workspace = true +litellm-secrets.workspace = true base64.workspace = true bytes.workspace = true data-url = "0.3.2" diff --git a/litellm-rust/crates/llms/src/aws_textract/ocr/analyze_transformation.rs b/litellm-rust/crates/llms/src/aws_textract/ocr/analyze_transformation.rs index d476861e6e1..2ce1b0da51b 100644 --- a/litellm-rust/crates/llms/src/aws_textract/ocr/analyze_transformation.rs +++ b/litellm-rust/crates/llms/src/aws_textract/ocr/analyze_transformation.rs @@ -40,6 +40,10 @@ impl BaseOcrConfig for TextractAnalyzeDocumentConfig { type ProviderRequest = AnalyzeDocumentRequest; type Environment = TextractEnvironment; + fn secret_names(&self) -> Vec<&'static str> { + litellm_auth_aws::constants::SECRET_NAMES.to_vec() + } + fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] { &["feature_types"] } diff --git a/litellm-rust/crates/llms/src/aws_textract/ocr/transformation.rs b/litellm-rust/crates/llms/src/aws_textract/ocr/transformation.rs index ad630a1ca4c..6eb195defaa 100644 --- a/litellm-rust/crates/llms/src/aws_textract/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/aws_textract/ocr/transformation.rs @@ -29,6 +29,10 @@ impl BaseOcrConfig for TextractDetectTextConfig { type ProviderRequest = DetectDocumentTextRequest; type Environment = TextractEnvironment; + fn secret_names(&self) -> Vec<&'static str> { + litellm_auth_aws::constants::SECRET_NAMES.to_vec() + } + fn get_health_check_document(&self) -> OcrDocument { health_check_document() } diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/cohere_parse_transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/cohere_parse_transformation.rs index 045d8744bc9..09639481cdf 100644 --- a/litellm-rust/crates/llms/src/azure_ai/ocr/cohere_parse_transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/cohere_parse_transformation.rs @@ -28,6 +28,10 @@ impl BaseOcrConfig for AzureAICohereParseConfig { super::transformation::AzureAiOcrConfig.get_api_key_env_var() } + fn secret_names(&self) -> Vec<&'static str> { + super::transformation::AzureAiOcrConfig.secret_names() + } + fn get_health_check_document(&self) -> OcrDocument { CohereParseConfig.get_health_check_document() } diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs index 9b27fdbb568..bfe0d76aab1 100644 --- a/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs @@ -2,7 +2,7 @@ use std::{collections::BTreeSet, time::Duration}; use base64::{Engine, engine::general_purpose::STANDARD}; use litellm_auth::{InputSource, Sourced}; -use litellm_auth_azure::AzureAuthInputs; +use litellm_auth_azure::{AzureAuthInputs, SECRET_NAMES as AZURE_AUTH_SECRET_NAMES}; use litellm_core_utils::{ call_arguments::CallArguments, serde_compat::{FiniteF64, LaxI64}, @@ -141,6 +141,17 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig { Some(AZURE_DI_API_KEY_ENV) } + fn secret_names(&self) -> Vec<&'static str> { + [ + [AZURE_DI_API_KEY_ENV, AZURE_DI_ENDPOINT_ENV].as_slice(), + AZURE_AUTH_SECRET_NAMES, + ] + .into_iter() + .flatten() + .copied() + .collect() + } + fn resolve_connection_params(&self, inputs: OcrCredentialInputs) -> ResolvedOcrCredentials { ResolvedOcrCredentials { api_key: inputs.api_key.and_then(|key| { diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs index 6df83e57eab..1fad860f757 100644 --- a/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs @@ -1,5 +1,6 @@ use litellm_auth::{InputSource, Sourced}; use litellm_auth_azure::AzureAuthInputs; +use litellm_auth_azure::SECRET_NAMES as AZURE_AUTH_SECRET_NAMES; use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl}; use serde_json::Value; @@ -37,6 +38,17 @@ impl BaseOcrConfig for AzureAiOcrConfig { Some(AZURE_AI_API_KEY_ENV) } + fn secret_names(&self) -> Vec<&'static str> { + [ + [AZURE_AI_API_KEY_ENV, AZURE_AI_API_BASE_ENV].as_slice(), + AZURE_AUTH_SECRET_NAMES, + ] + .into_iter() + .flatten() + .copied() + .collect() + } + fn map_ocr_params( &self, non_default_params: &CallArguments, diff --git a/litellm-rust/crates/llms/src/base_llm/inference/mod.rs b/litellm-rust/crates/llms/src/base_llm/inference/mod.rs new file mode 100644 index 00000000000..10c0454f947 --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/inference/mod.rs @@ -0,0 +1 @@ +pub mod secrets; diff --git a/litellm-rust/crates/llms/src/base_llm/inference/secrets.rs b/litellm-rust/crates/llms/src/base_llm/inference/secrets.rs new file mode 100644 index 00000000000..eb13fe95116 --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/inference/secrets.rs @@ -0,0 +1,19 @@ +use std::sync::Arc; + +use futures_util::future::BoxFuture; +use litellm_core_utils::settings::{Lookup, ProcessEnvironment}; +use litellm_secrets::Error; + +pub type Secrets = Arc; + +pub trait SecretSource: Send + Sync { + fn resolve<'a>(&'a self, names: &'a [&'static str]) -> BoxFuture<'a, Result>; +} + +pub struct EnvironmentSecrets; + +impl SecretSource for EnvironmentSecrets { + fn resolve<'a>(&'a self, _names: &'a [&'static str]) -> BoxFuture<'a, Result> { + Box::pin(async { Ok(Arc::new(ProcessEnvironment) as Secrets) }) + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/mod.rs b/litellm-rust/crates/llms/src/base_llm/mod.rs index 8ed37da4573..9cced64b687 100644 --- a/litellm-rust/crates/llms/src/base_llm/mod.rs +++ b/litellm-rust/crates/llms/src/base_llm/mod.rs @@ -2,5 +2,6 @@ pub mod anthropic_messages; pub mod audio_transcription; pub mod base_model_iterator; pub mod chat; +pub mod inference; pub mod ocr; pub mod responses; diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/error.rs b/litellm-rust/crates/llms/src/base_llm/ocr/error.rs index e09842e2856..b3df8fc18c8 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/error.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/error.rs @@ -98,6 +98,8 @@ pub enum Error { "Missing REDUCTO_API_KEY - set it in the environment or pass api_key to litellm.ocr()/litellm.aocr()" )] MissingReductoApiKey, + #[error("secret resolution failed: {0}")] + Secret(#[source] std::sync::Arc), #[error(transparent)] Auth(#[from] litellm_auth::Error), #[error(transparent)] diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/handler.rs b/litellm-rust/crates/llms/src/base_llm/ocr/handler.rs index 245261d9f92..3ec9de8197f 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/handler.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/handler.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use bytes::{Bytes, BytesMut}; use futures_util::future::BoxFuture; use litellm_auth_gcp::VertexAuth; @@ -11,9 +13,10 @@ use litellm_http::{ use serde::{Serialize, de::DeserializeOwned}; use serde_json::Value; +use crate::base_llm::inference::secrets::SecretSource; use crate::base_llm::ocr::{ error::Error, - settings::{OcrSettings, Secrets}, + settings::OcrSettings, transformation::{ BaseOcrConfig, DecodedOcrResponse, LiteLLMOcrResponse, OcrDocument, OcrResponseContext, PreparedOcrRequest, decode_request_value, decode_response, @@ -35,7 +38,7 @@ pub struct OcrClient { document_fetcher: MediaFetcher, vertex_auth: VertexAuth, settings: OcrSettings, - secrets: Secrets, + secrets: Arc, } impl OcrClient { @@ -45,7 +48,7 @@ impl OcrClient { url_policy: UrlPolicy, vertex_auth: VertexAuth, settings: OcrSettings, - secrets: Secrets, + secrets: Arc, ) -> Result { Ok(Self { provider_http: pool.client(config, ClientVariant::Provider)?, @@ -77,7 +80,7 @@ impl OcrClient { &self.settings } - pub fn secrets(&self) -> &Secrets { + pub fn secret_source(&self) -> &Arc { &self.secrets } @@ -92,7 +95,7 @@ impl OcrClient { document_fetcher: MediaFetcher::for_test(document_http), vertex_auth: VertexAuth::default(), settings: OcrSettings::default(), - secrets: std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment), + secrets: Arc::new(crate::base_llm::inference::secrets::EnvironmentSecrets), } } @@ -102,7 +105,7 @@ impl OcrClient { } #[cfg(any(test, feature = "test-support"))] - pub fn with_secrets(self, secrets: Secrets) -> Self { + pub fn with_secrets(self, secrets: Arc) -> Self { Self { secrets, ..self } } } diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/settings.rs b/litellm-rust/crates/llms/src/base_llm/ocr/settings.rs index f5954599b43..87461cd36aa 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/settings.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/settings.rs @@ -1,9 +1,7 @@ -use std::{sync::Arc, time::Duration}; +use std::time::Duration; use litellm_core_utils::settings::Lookup; -pub type Secrets = Arc; - #[derive(Clone, Debug, PartialEq)] pub struct OcrSettings { pub request_timeout: Duration, diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs index e02a4b7f266..0506ff3d6df 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs @@ -14,10 +14,13 @@ use serde::{ use serde_json::{Map, Value}; use serde_with::serde_as; -use crate::base_llm::ocr::{ - error::Error, - handler::{CallHooks, OcrClient, read_response_bytes, transform_request_body}, - settings::{OcrSettings, Secrets}, +use crate::base_llm::{ + inference::secrets::Secrets, + ocr::{ + error::Error, + handler::{CallHooks, OcrClient, read_response_bytes, transform_request_body}, + settings::OcrSettings, + }, }; pub const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024; @@ -436,6 +439,8 @@ pub trait BaseOcrConfig: Send + Sync + Sized + 'static { None } + fn secret_names(&self) -> Vec<&'static str>; + fn resolve_connection_params(&self, inputs: OcrCredentialInputs) -> ResolvedOcrCredentials { ResolvedOcrCredentials { api_key: inputs diff --git a/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs b/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs index d141c68db38..c0bb4c60563 100644 --- a/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs @@ -102,6 +102,10 @@ impl BaseOcrConfig for CohereParseConfig { Some(COHERE_API_KEY_ENV) } + fn secret_names(&self) -> Vec<&'static str> { + vec![COHERE_API_KEY_ENV] + } + fn get_health_check_document(&self) -> OcrDocument { OcrDocument::ImageUrl { image_url: COHERE_PARSE_HEALTH_CHECK_IMAGE_DATA_URI.into(), diff --git a/litellm-rust/crates/llms/src/mistral/ocr/transformation.rs b/litellm-rust/crates/llms/src/mistral/ocr/transformation.rs index 2b14372fbec..149e8056789 100644 --- a/litellm-rust/crates/llms/src/mistral/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/mistral/ocr/transformation.rs @@ -69,6 +69,14 @@ impl BaseOcrConfig for MistralOcrConfig { Some(MISTRAL_OCR_API_KEY_ENV_VAR) } + fn secret_names(&self) -> Vec<&'static str> { + vec![ + MISTRAL_OCR_API_KEY_ENV_VAR, + "MISTRAL_AZURE_API_KEY", + "MISTRAL_AZURE_API_BASE", + ] + } + fn map_ocr_params( &self, non_default_params: &CallArguments, diff --git a/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs b/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs index 5272be97c24..f00259984ba 100644 --- a/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs @@ -92,6 +92,10 @@ impl BaseOcrConfig for ReductoParseV3Config { type ProviderRequest = ReductoV3Request; type Environment = Vec<(String, String)>; + fn secret_names(&self) -> Vec<&'static str> { + vec![REDUCTO_API_KEY_ENV] + } + fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] { &["formatting", "retrieval", "settings"] } @@ -180,6 +184,10 @@ impl BaseOcrConfig for ReductoParseLegacyConfig { type ProviderRequest = ReductoLegacyRequest; type Environment = Vec<(String, String)>; + fn secret_names(&self) -> Vec<&'static str> { + vec![REDUCTO_API_KEY_ENV] + } + fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] { &["enhance"] } diff --git a/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs index 9a23deefb89..86231d50f9c 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs @@ -105,6 +105,10 @@ impl BaseOcrConfig for VertexAIDeepSeekOCRConfig { VertexAiOcrConfig.get_api_key_env_var() } + fn secret_names(&self) -> Vec<&'static str> { + VertexAiOcrConfig.secret_names() + } + fn map_ocr_params( &self, _arguments: &CallArguments, diff --git a/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs index 2d505ba4342..c9342c87e9a 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs @@ -34,6 +34,10 @@ impl BaseOcrConfig for VertexAiOcrConfig { Some("VERTEX_AI_API_KEY") } + fn secret_names(&self) -> Vec<&'static str> { + litellm_auth_gcp::SECRET_NAMES.to_vec() + } + fn map_ocr_params( &self, non_default_params: &CallArguments, diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 6b4476d897c..dd0975219e4 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -10,7 +10,7 @@ name = "_native" crate-type = ["cdylib"] [features] -default = ["abi3", "fast"] +default = ["abi3", "fast", "huggingface", "tiktoken"] abi3 = ["pyo3/abi3-py310"] extension-module = ["pyo3/extension-module"] panic-test = [] @@ -20,6 +20,7 @@ tiktoken = ["litellm-token-counter/tiktoken"] [dependencies] bytes.workspace = true +futures-util.workspace = true litellm-cache.workspace = true litellm-cache-azure-blob.workspace = true litellm-cache-memory.workspace = true @@ -41,6 +42,8 @@ litellm-core-utils.workspace = true litellm-auth-gcp.workspace = true litellm-http.workspace = true litellm-llms.workspace = true +litellm-secrets = { workspace = true, features = ["aws"] } +litellm-secrets-types.workspace = true litellm-types.workspace = true litellm-host-python.workspace = true litellm-token-counter = { path = "../token-counter", default-features = false } @@ -49,10 +52,12 @@ pyo3-async-runtimes.workspace = true reqwest.workspace = true redis = { version = "1.7.0", features = ["tls-rustls"] } serde_json.workspace = true -url.workspace = true +thiserror.workspace = true tokio = { workspace = true, features = ["rt", "sync"] } +url.workspace = true [dev-dependencies] +litellm-secrets-aws.workspace = true serde.workspace = true serde_with.workspace = true criterion.workspace = true @@ -60,6 +65,8 @@ futures-util.workspace = true rstest.workspace = true sha2.workspace = true tokio-tungstenite.workspace = true +wiremock = "0.6.5" +aws-sdk-secretsmanager = "1.117.0" [[bench]] name = "serialization" diff --git a/litellm-rust/crates/python-bridge/README.md b/litellm-rust/crates/python-bridge/README.md new file mode 100644 index 00000000000..faaca233f5a --- /dev/null +++ b/litellm-rust/crates/python-bridge/README.md @@ -0,0 +1,5 @@ +Native OCR uses `SecretSource` with `EnvironmentSecrets`, preserving process-environment reads. Readable Python secret managers still make OCR decline to the existing Python implementation. `ResolvedSecrets` and the separate `secret_manager_binding()` snapshot are inactive foundations for a later rollout + +Cache and secret-manager catalog entries remain Python-only, including when `LITELLM_RUST=1`. The new cache runtime is not connected to SDK or gateway caching + +OCR provider requests use the shared `litellm-http` pool. AWS and Google secret-manager SDK clients keep their SDK transports, which do not yet inherit the pool's proxy, TLS, certificate, timeout, or observability configuration. Preserve those SDK transports and configure them equivalently instead of forcing them through reqwest diff --git a/litellm-rust/crates/python-bridge/python_settings.json b/litellm-rust/crates/python-bridge/python_settings.json deleted file mode 100644 index ea53d1d2025..00000000000 --- a/litellm-rust/crates/python-bridge/python_settings.json +++ /dev/null @@ -1,154 +0,0 @@ -{ - "http_settings": { - "version": 1, - "fields": { - "ssl_verify": { - "adapter": "SslVerifyInput", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [ - "none", - "bool", - "str" - ], - "unsupported_live": "configuration_error" - }, - "ssl_certificate": { - "adapter": "OptionalStrictString", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "ssl_security_level": { - "adapter": "TuningString", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "ssl_ecdh_curve": { - "adapter": "TuningString", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "force_ipv4": { - "adapter": "Truthy", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "http2": { - "adapter": "ExactTrue", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "aiohttp_trust_env": { - "adapter": "Truthy", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "disable_aiohttp_trust_env": { - "adapter": "Truthy", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "disable_aiohttp_transport": { - "adapter": "ExactTrue", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "user_agent": { - "adapter": "StrictString", - "required": true, - "precedence": "accessor", - "sensitive": false, - "shapes": [], - "unsupported_live": null - } - } - }, - "url_policy": { - "version": 1, - "fields": { - "user_url_validation": { - "adapter": "Truthy", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - }, - "user_url_allowed_hosts": { - "adapter": "HostCollection", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - } - } - }, - "provider_defaults": { - "version": 1, - "fields": { - "vertex_project": { - "adapter": "FalsyOptionalString", - "required": true, - "precedence": "module_global", - "sensitive": true, - "shapes": [], - "unsupported_live": null - }, - "vertex_location": { - "adapter": "FalsyOptionalString", - "required": true, - "precedence": "module_global", - "sensitive": true, - "shapes": [], - "unsupported_live": null - }, - "enable_azure_ad_token_refresh": { - "adapter": "ExactTrue", - "required": true, - "precedence": "module_global", - "sensitive": false, - "shapes": [], - "unsupported_live": null - } - } - }, - "secret_manager": { - "version": 1, - "fields": { - "readable": { - "adapter": "StrictBool", - "required": true, - "precedence": "accessor", - "sensitive": false, - "shapes": [], - "unsupported_live": null - } - } - } -} diff --git a/litellm-rust/crates/python-bridge/src/cache/activation.rs b/litellm-rust/crates/python-bridge/src/cache/activation.rs new file mode 100644 index 00000000000..8d2c340dec8 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/cache/activation.rs @@ -0,0 +1,111 @@ +use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig}; +use litellm_cache_redis_semantic::RedisSemanticConfig; +use litellm_host_python::{release_gil, run_sync_value}; +use litellm_http::ClientVariant; +use pyo3::prelude::*; + +use super::{ + cache_error, + config::{CacheBackendConfig, NativeCacheConfig, UnsupportedCacheConfig}, + embedder::PythonEmbedder, + host_client, + native::NativeResponseCache, +}; +use crate::errors::RustBridgeDeclined; + +fn declined(reason: UnsupportedCacheConfig) -> PyErr { + RustBridgeDeclined::new_err(reason.message()) +} + +/// Builds the native backend a `Cache` facade's projected configuration describes. `backend` is +/// the facade's `.cache` object, which owns embedding for the Python-embedded semantic caches. +pub(super) fn activate( + py: Python<'_>, + backend: &Bound<'_, PyAny>, + config: NativeCacheConfig, +) -> PyResult { + let policy = config.policy; + let service = match config.backend { + CacheBackendConfig::Memory(memory) => { + NativeResponseCache::memory(memory.capacity, memory.default_ttl, memory.max_entry_bytes) + } + CacheBackendConfig::Redis(redis) => { + let url = redis.connection.native_url().map_err(declined)?; + let flush_size = policy.redis_flush_size.map(|_| redis.flush_size); + release_gil(py, move || { + NativeResponseCache::redis( + &url, + &redis.topology, + Some(redis.default_ttl), + redis.namespace, + ) + }) + .map_err(cache_error)? + .with_redis_flush_size(flush_size) + } + CacheBackendConfig::S3(s3) => { + let http = host_client(py, ClientVariant::NoRedirect)?; + run_sync_value( + py, + async move { Ok(NativeResponseCache::s3(*s3, http).await) }, + )? + } + CacheBackendConfig::Gcs(gcs) => NativeResponseCache::gcs( + GcsConfig { + bucket_name: gcs.bucket_name, + gcs_path: Some(gcs.key_prefix), + path_service_account: gcs.path_service_account, + endpoint: DEFAULT_ENDPOINT.to_owned(), + }, + host_client(py, ClientVariant::NoRedirect)?, + None, + ), + CacheBackendConfig::Disk(disk) => { + release_gil(py, move || NativeResponseCache::disk(&disk.directory)) + .map_err(cache_error)? + } + CacheBackendConfig::AzureBlob(azure) => { + let http = host_client(py, ClientVariant::NoRedirect)?; + run_sync_value(py, async move { + NativeResponseCache::azure_blob(&azure.account_url, &azure.container, http) + .await + .map_err(cache_error) + })? + } + CacheBackendConfig::RedisSemantic(semantic) => { + let url = semantic.native_url().map_err(declined)?.to_owned(); + let embedder = PythonEmbedder::new(backend.clone().unbind()); + let semantic_config = RedisSemanticConfig { + index_name: semantic.index_name, + similarity_threshold: semantic.similarity_threshold as f32, + }; + release_gil(py, move || { + NativeResponseCache::redis_semantic(&url, embedder, semantic_config) + }) + .map_err(cache_error)? + } + CacheBackendConfig::ValkeySemantic(valkey) => { + let url = valkey.connection.native_url().map_err(declined)?; + let embedder = PythonEmbedder::new(backend.clone().unbind()); + release_gil(py, move || { + NativeResponseCache::valkey_semantic( + &url, + valkey.similarity_threshold, + valkey.index_name, + embedder, + ) + }) + .map_err(cache_error)? + } + CacheBackendConfig::QdrantSemantic(qdrant) => { + let client = host_client(py, ClientVariant::Provider)?; + run_sync_value(py, async move { + let runtime = tokio::runtime::Handle::current(); + NativeResponseCache::qdrant_semantic(*qdrant, client, runtime) + .await + .map_err(cache_error) + })? + } + }; + Ok(service.with_scope(policy.semantic_cache_scope)) +} diff --git a/litellm-rust/crates/python-bridge/src/cache/binding.rs b/litellm-rust/crates/python-bridge/src/cache/binding.rs index 2ff73238202..56f1bff2853 100644 --- a/litellm-rust/crates/python-bridge/src/cache/binding.rs +++ b/litellm-rust/crates/python-bridge/src/cache/binding.rs @@ -9,12 +9,15 @@ use pyo3::{ use serde_json::Value; use super::{ + activation::activate, cache_error, callback::PythonCallback, + config::{CacheConfigProjection, NativeCacheConfig}, future::{ready_none, ready_value}, - native::NativeResponseCache, + native::{NativeResponseCache, SemanticReply}, request::{now, request, requests}, }; +use crate::errors::RustBridgeDeclined; pub(super) enum CacheBinding { Disabled, @@ -22,7 +25,7 @@ pub(super) enum CacheBinding { PythonCallback(PythonCallback), } -#[pyclass(frozen, name = "_CacheTestBinding")] +#[pyclass(frozen, name = "_ResponseCacheRuntime")] pub(crate) struct ResolvedCache { binding: CacheBinding, pid: u32, @@ -66,6 +69,19 @@ impl ResolvedCache { #[pymethods] impl ResolvedCache { + #[staticmethod] + fn from_cache(cache: &Bound<'_, PyAny>) -> PyResult { + let config = match NativeCacheConfig::project(cache)? { + CacheConfigProjection::Native(config) => *config, + CacheConfigProjection::Unsupported(reason) => { + return Err(RustBridgeDeclined::new_err(reason.message())); + } + }; + let backend = cache.getattr("cache")?; + let service = activate(cache.py(), &backend, config)?; + Ok(Self::new(CacheBinding::Native(service))) + } + #[getter] fn kind(&self) -> &'static str { match self.binding { @@ -98,6 +114,41 @@ impl ResolvedCache { } } + /// `(response, similarity)`: the similarity is `None` when the backend reports none. + fn lookup_semantic(&self, py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult> { + self.check_process()?; + match &self.binding { + CacheBinding::Native(service) => { + let request = self::request(request)?; + let service = service.clone(); + let lookup = release_gil(py, move || service.lookup_semantic(&request, now())) + .map_err(cache_error)?; + to_py(py, &SemanticReply::from(lookup)) + } + CacheBinding::Disabled => to_py(py, &SemanticReply(None, None)), + CacheBinding::PythonCallback(_) => Err(PyRuntimeError::new_err( + "semantic lookups require a native cache binding", + )), + } + } + + fn async_lookup_semantic<'py>( + &self, + py: Python<'py>, + request: &Bound<'py, PyAny>, + ) -> PyResult> { + self.check_process()?; + match &self.binding { + CacheBinding::Native(service) => { + service.async_lookup_semantic_py(py, self::request(request)?) + } + CacheBinding::Disabled => ready_value(py, &SemanticReply(None, None)), + CacheBinding::PythonCallback(_) => Err(PyRuntimeError::new_err( + "semantic lookups require a native cache binding", + )), + } + } + #[pyo3(signature = (request, response, *, callback_kwargs=None))] fn store( &self, diff --git a/litellm-rust/crates/python-bridge/src/cache/config.rs b/litellm-rust/crates/python-bridge/src/cache/config.rs index d161b401928..6e25f07efa1 100644 --- a/litellm-rust/crates/python-bridge/src/cache/config.rs +++ b/litellm-rust/crates/python-bridge/src/cache/config.rs @@ -11,14 +11,9 @@ use pyo3::{ types::{PyAny, PyBool, PyDict, PyList, PyString}, }; -use super::{native::NativeResponseCache, request::duration}; +use super::{identity::BackendIdentity, native::NativeResponseCache, request::duration}; -#[allow(dead_code, reason = "consumed by the cache activation follow-up")] pub(super) struct CachePolicy { - pub(super) mode: String, - pub(super) ttl: Option, - pub(super) namespace: Option, - pub(super) supported_call_types: Option>, pub(super) redis_flush_size: Option, pub(super) semantic_cache_scope: String, } @@ -46,7 +41,6 @@ pub(super) enum CertificateRequirement { Required, } -#[allow(dead_code, reason = "consumed by the cache activation follow-up")] pub(super) struct RedisTlsConfig { pub(super) certificate_requirement: CertificateRequirement, pub(super) check_hostname: bool, @@ -56,7 +50,6 @@ pub(super) struct RedisTlsConfig { pub(super) client_key: Option, } -#[allow(dead_code, reason = "consumed by the cache activation follow-up")] pub(super) struct RedisConnectionConfig { pub(super) host: String, pub(super) port: u16, @@ -73,7 +66,6 @@ pub(super) struct RedisConnectionConfig { pub(super) tls: Option, } -#[allow(dead_code, reason = "consumed by the cache activation follow-up")] pub(super) struct RedisCacheConfig { pub(super) default_ttl: Duration, pub(super) namespace: Option, @@ -94,17 +86,10 @@ pub(super) struct AzureBlobCacheConfig { pub(super) container: String, } -#[allow( - dead_code, - reason = "embedding settings are projected so drift falls back to Python" -)] pub(super) struct RedisSemanticCacheConfig { pub(super) redis_url: String, pub(super) index_name: String, pub(super) similarity_threshold: f64, - pub(super) embedding_model: String, - pub(super) embedding_max_input_tokens: Option, - pub(super) embedding_timeout: Option, } struct RedisClientProjection<'py> { @@ -118,11 +103,13 @@ struct RedisClientProjection<'py> { const REDIS_PY_DEFAULT_MAX_CONNECTIONS: usize = 1 << 31; -#[allow(dead_code, reason = "consumed by the cache activation follow-up")] +/// The read and write timeout every native Redis connection uses, which is also `RedisCache`'s +/// default `socket_timeout`. +const NATIVE_REDIS_SOCKET_TIMEOUT: Duration = Duration::from_secs(5); + pub(super) struct ValkeySemanticCacheConfig { pub(super) similarity_threshold: f64, pub(super) index_name: String, - pub(super) embedding_model: String, pub(super) connection: RedisConnectionConfig, } @@ -147,6 +134,93 @@ impl QdrantSemanticCacheConfig { } } +impl RedisTlsConfig { + /// Whether redis-rs with rustls behaves like this redis-py `SSLConnection`: it verifies the + /// certificate chain against the system roots and always checks the hostname. + fn native(&self) -> Result<(), UnsupportedCacheConfig> { + if self.ca_certificate.is_some() + || self.ca_data.is_some() + || self.client_certificate.is_some() + || self.client_key.is_some() + { + return Err(UnsupportedCacheConfig::RedisTlsCertificates); + } + if self.certificate_requirement == CertificateRequirement::None || !self.check_hostname { + return Err(UnsupportedCacheConfig::RedisTlsVerification); + } + Ok(()) + } +} + +impl RedisConnectionConfig { + /// The redis-rs URL for this connection, or the first setting the native client cannot + /// honor. The native pool and socket timeouts are fixed, so only redis-py's unbounded pool, + /// its unset timeouts and `RedisCache`'s five-second `socket_timeout` map onto them. + pub(super) fn native_url(&self) -> Result { + if self.pool_size != REDIS_PY_DEFAULT_MAX_CONNECTIONS { + return Err(UnsupportedCacheConfig::RedisPoolSize); + } + if self + .read_timeout + .is_some_and(|timeout| timeout != NATIVE_REDIS_SOCKET_TIMEOUT) + || self.connect_timeout.is_some() + { + return Err(UnsupportedCacheConfig::RedisTimeout); + } + if self.socket_keepalive == Some(true) { + return Err(UnsupportedCacheConfig::RedisKeepalive); + } + if !self.health_check_interval.is_zero() { + return Err(UnsupportedCacheConfig::RedisHealthCheck); + } + if self.client_name.is_some() { + return Err(UnsupportedCacheConfig::RedisClientName); + } + let scheme = match &self.tls { + None => "redis", + Some(tls) => { + tls.native()?; + "rediss" + } + }; + let host = if self.host.contains(':') { + format!("[{}]", self.host) + } else { + self.host.clone() + }; + let mut url = url::Url::parse(&format!( + "{scheme}://{host}:{}/{}", + self.port, self.database + )) + .map_err(|_| UnsupportedCacheConfig::RedisConnection)?; + if let Some(username) = &self.username { + url.set_username(username) + .map_err(|()| UnsupportedCacheConfig::RedisConnection)?; + } + if let Some(password) = &self.password { + url.set_password(Some(password)) + .map_err(|()| UnsupportedCacheConfig::RedisConnection)?; + } + if self.protocol == RedisProtocol::Resp3 { + url.set_query(Some("protocol=resp3")); + } + Ok(url.into()) + } +} + +impl RedisSemanticCacheConfig { + /// redisvl hands `redis_url` to redis-py, which reads TLS and socket options from the URL; + /// redis-rs ignores those, so only a plain URL keeps its meaning. + pub(super) fn native_url(&self) -> Result<&str, UnsupportedCacheConfig> { + let url = url::Url::parse(&self.redis_url) + .map_err(|_| UnsupportedCacheConfig::RedisSemanticUrl)?; + if !matches!(url.scheme(), "redis" | "unix") || url.query().is_some() { + return Err(UnsupportedCacheConfig::RedisSemanticUrl); + } + Ok(&self.redis_url) + } +} + pub(super) enum CacheBackendConfig { Memory(MemoryCacheConfig), Redis(Box), @@ -159,7 +233,6 @@ pub(super) enum CacheBackendConfig { QdrantSemantic(Box), } -#[allow(dead_code, reason = "consumed by the cache activation follow-up")] pub(super) struct NativeCacheConfig { pub(super) policy: CachePolicy, pub(super) backend: CacheBackendConfig, @@ -178,6 +251,15 @@ pub(super) enum UnsupportedCacheConfig { DiskStore, QdrantEndpoint, SemanticEmbedding, + RedisPoolSize, + RedisTimeout, + RedisKeepalive, + RedisHealthCheck, + RedisClientName, + RedisTlsCertificates, + RedisTlsVerification, + RedisSemanticUrl, + ValkeyTls, } impl UnsupportedCacheConfig { @@ -197,6 +279,28 @@ impl UnsupportedCacheConfig { "native Qdrant requires the default REST port so the gRPC port can be derived" } Self::SemanticEmbedding => "native semantic embedding requires Python", + Self::RedisPoolSize => { + "native Redis uses a fixed connection pool; max_connections requires Python" + } + Self::RedisTimeout => { + "native Redis uses fixed socket timeouts; socket_timeout and \ + socket_connect_timeout require Python" + } + Self::RedisKeepalive => "native Redis does not support socket_keepalive", + Self::RedisHealthCheck => "native Redis does not support health_check_interval", + Self::RedisClientName => "native Redis does not support client_name", + Self::RedisTlsCertificates => { + "native Redis TLS does not support ssl_ca_certs, ssl_ca_data, ssl_certfile or \ + ssl_keyfile" + } + Self::RedisTlsVerification => { + "native Redis TLS always verifies the certificate and hostname; \ + ssl_cert_reqs=none and ssl_check_hostname=false require Python" + } + Self::RedisSemanticUrl => { + "native Redis semantic cache does not support TLS or query options in redis_url" + } + Self::ValkeyTls => "native Valkey semantic cache does not support TLS connections", } } } @@ -211,12 +315,6 @@ impl NativeCacheConfig { pub(super) fn project(facade: &Bound<'_, PyAny>) -> PyResult { let backend_name = facade.getattr("type")?.extract::()?; let policy = CachePolicy { - mode: facade.getattr("mode")?.extract::()?, - ttl: optional_duration(facade.getattr("ttl")?)?, - namespace: optional_string(facade.getattr("namespace")?)?, - supported_call_types: facade - .getattr("supported_call_types")? - .extract::>>()?, redis_flush_size: facade .getattr("redis_flush_size")? .extract::>()?, @@ -293,160 +391,58 @@ impl NativeCacheConfig { } pub(super) fn service_mismatch(&self, service: &NativeResponseCache) -> Option<&'static str> { - let default_ttl = match &self.backend { - CacheBackendConfig::Memory(config) => Some(config.default_ttl), - CacheBackendConfig::Redis(config) => Some(config.default_ttl), - CacheBackendConfig::S3(_) => None, - CacheBackendConfig::ValkeySemantic(_) => Some(Duration::ZERO), - CacheBackendConfig::Disk(_) - | CacheBackendConfig::AzureBlob(_) - | CacheBackendConfig::Gcs(_) - | CacheBackendConfig::RedisSemantic(_) - | CacheBackendConfig::QdrantSemantic(_) => None, - }; - if !matches!(self.backend, CacheBackendConfig::ValkeySemantic(_)) - && service.default_ttl() != default_ttl - { - return Some("facade and native backend default TTLs must match"); - } - match &self.backend { - CacheBackendConfig::Memory(config) if service.kind() != "memory" => { - Some("facade and native backend types must match") - } - CacheBackendConfig::Memory(config) if service.capacity() != Some(config.capacity) => { - Some("facade and native backend capacities must match") - } - CacheBackendConfig::Memory(config) - if service.max_entry_bytes() != Some(config.max_entry_bytes) => - { - Some("facade and native backend item limits must match") - } - CacheBackendConfig::Memory(_) => None, - CacheBackendConfig::Redis(_) if service.kind() != "redis" => { - Some("facade and native backend types must match") - } - CacheBackendConfig::Redis(config) if service.topology() != Some(&config.topology) => { - Some("facade and native backend topologies must match") - } - CacheBackendConfig::Redis(config) => (service.namespace() - != config.namespace.as_deref()) - .then_some("facade and native backend namespaces must match"), - CacheBackendConfig::S3(_) if service.kind() != "s3" => { - Some("facade and native backend types must match") - } - CacheBackendConfig::S3(config) if service.bucket() != Some(config.bucket.as_str()) => { - Some("facade and native backend buckets must match") - } - CacheBackendConfig::S3(config) - if service.key_prefix() != Some(config.key_prefix.as_str()) => - { - Some("facade and native backend key prefixes must match") - } - CacheBackendConfig::S3(config) if service.region() != Some(config.region.as_str()) => { - Some("facade and native backend regions must match") - } - CacheBackendConfig::S3(config) - if service.endpoint() - != config - .endpoint - .as_ref() - .map(|endpoint| endpoint.url.as_str()) => - { - Some("facade and native backend endpoints must match") - } - CacheBackendConfig::S3(_) => None, - CacheBackendConfig::Gcs(_) if service.kind() != "gcs" => { - Some("facade and native backend types must match") - } - CacheBackendConfig::Gcs(config) - if service - .gcs_backend() - .is_none_or(|backend| backend.bucket_name() != config.bucket_name) => - { - Some("facade and native backend buckets must match") - } - CacheBackendConfig::Gcs(config) - if service - .gcs_backend() - .is_none_or(|backend| backend.key_prefix() != config.key_prefix) => - { - Some("facade and native backend key prefixes must match") - } - CacheBackendConfig::Gcs(config) - if service.gcs_backend().is_none_or(|backend| { - backend.path_service_account() != config.path_service_account.as_deref() - }) => - { - Some("facade and native backend credentials must match") - } - CacheBackendConfig::Gcs(_) => None, - CacheBackendConfig::ValkeySemantic(config) => { - if service.kind() != "valkey-semantic" { - return Some("facade and native backend types must match"); - } - let Some((threshold, index_name)) = service.semantic_config() else { - return Some("facade and native backend types must match"); - }; - (threshold != config.similarity_threshold || index_name != config.index_name) - .then_some("facade and native semantic settings must match") - } - CacheBackendConfig::Disk(_) if service.kind() != "disk" => { - Some("facade and native backend types must match") - } - CacheBackendConfig::Disk(config) => { - let Some(directory) = service.directory() else { - return Some("facade and native backend types must match"); - }; - let native = std::fs::canonicalize(directory).ok(); - let facade = std::fs::canonicalize(&config.directory).ok(); - (native != facade).then_some("facade and native backend directories must match") - } - CacheBackendConfig::RedisSemantic(_) if service.kind() != "redis_semantic" => { - Some("facade and native backend types must match") - } - CacheBackendConfig::RedisSemantic(config) - if service.index_name() != Some(config.index_name.as_str()) => - { - Some("facade and native backend index names must match") - } - CacheBackendConfig::RedisSemantic(config) - if service.similarity_threshold() != Some(config.similarity_threshold) => - { - Some("facade and native backend similarity thresholds must match") - } - CacheBackendConfig::RedisSemantic(_) => None, - CacheBackendConfig::QdrantSemantic(config) if service.kind() != "qdrant_semantic" => { - Some("facade and native backend types must match") - } - CacheBackendConfig::QdrantSemantic(config) - if service.collection_name() != Some(config.collection_name.as_str()) => - { - Some("facade and native backend collections must match") - } - CacheBackendConfig::QdrantSemantic(config) - if service.similarity_threshold() != Some(config.similarity_threshold) => - { - Some("facade and native backend similarity thresholds must match") - } - CacheBackendConfig::QdrantSemantic(config) - if service.vector_size() != Some(config.vector_size) => - { - Some("facade and native backend vector sizes must match") - } - CacheBackendConfig::QdrantSemantic(config) - if service.embedding_model() != Some(config.embedding.model.as_str()) => - { - Some("facade and native backend embedding models must match") - } - CacheBackendConfig::QdrantSemantic(_) => None, - CacheBackendConfig::AzureBlob(config) => match service.azure_blob_identity() { - None => Some("facade and native backend types must match"), - Some((account_url, container)) - if account_url != config.account_url || container != config.container => - { - Some("facade and native backend containers must match") - } - Some(_) => None, + self.backend.identity().mismatch(&service.identity()) + } +} + +impl CacheBackendConfig { + /// The identity a native backend must have for this facade configuration to describe it. + pub(super) fn identity(&self) -> BackendIdentity { + match self { + Self::Memory(config) => BackendIdentity::Memory { + capacity: config.capacity, + max_entry_bytes: Some(config.max_entry_bytes), + default_ttl: Some(config.default_ttl), + }, + Self::Redis(config) => BackendIdentity::Redis { + topology: config.topology.clone(), + namespace: config.namespace.clone(), + default_ttl: Some(config.default_ttl), + }, + Self::S3(config) => BackendIdentity::S3 { + bucket: config.bucket.clone(), + key_prefix: config.key_prefix.clone(), + region: config.region.clone(), + endpoint: config + .endpoint + .as_ref() + .map(|endpoint| endpoint.url.clone()), + }, + Self::Gcs(config) => BackendIdentity::Gcs { + bucket_name: config.bucket_name.clone(), + key_prefix: config.key_prefix.clone(), + path_service_account: config.path_service_account.clone(), + }, + Self::ValkeySemantic(config) => BackendIdentity::ValkeySemantic { + index_name: config.index_name.clone(), + similarity_threshold: config.similarity_threshold, + }, + Self::Disk(config) => BackendIdentity::Disk { + directory: config.directory.clone(), + }, + Self::AzureBlob(config) => BackendIdentity::AzureBlob { + account_url: config.account_url.clone(), + container: config.container.clone(), + }, + Self::RedisSemantic(config) => BackendIdentity::RedisSemantic { + index_name: config.index_name.clone(), + similarity_threshold: config.similarity_threshold as f32, + }, + Self::QdrantSemantic(config) => BackendIdentity::QdrantSemantic { + collection_name: config.collection_name.clone(), + similarity_threshold: config.similarity_threshold, + vector_size: config.vector_size, + embedding_model: config.embedding.model.clone(), }, } } @@ -577,13 +573,6 @@ pub(super) fn project_redis_semantic( .extract::>()? .unwrap_or_else(|| "litellm_semantic_cache_index".into()), similarity_threshold: backend.getattr("similarity_threshold")?.extract::()?, - embedding_model: backend.getattr("embedding_model")?.extract::()?, - embedding_max_input_tokens: backend - .getattr("embedding_max_input_tokens")? - .extract::>()?, - embedding_timeout: backend - .getattr("embedding_timeout")? - .extract::>()?, }) } @@ -694,37 +683,48 @@ fn project_redis( if has_value(&resolved, "credential_provider")? { return Ok(Err(UnsupportedCacheConfig::RedisCredentials)); } - - let protocol = match optional_i64(&resolved, "protocol")?.unwrap_or(2) { - 2 => RedisProtocol::Resp2, - 3 => RedisProtocol::Resp3, - _ => return Err(PyValueError::new_err("unsupported Redis protocol version")), - }; - let health_check_interval = - duration(optional_f64(&resolved, "health_check_interval")?.unwrap_or(0.0))?; Ok(Ok(RedisCacheConfig { default_ttl: duration(backend.getattr("default_ttl")?.extract::()?)?, namespace: optional_attribute_string(backend, "namespace")?, flush_size: backend.getattr("redis_flush_size")?.extract::()?, topology, - connection: RedisConnectionConfig { - host, - port, - database: optional_i64(&resolved, "db")?.unwrap_or(0), - username: optional_dict_string(&resolved, "username")?, - password: optional_dict_string(&resolved, "password")?, - protocol, - pool_size, - read_timeout: optional_dict_duration(&resolved, "socket_timeout")?, - connect_timeout: optional_dict_duration(&resolved, "socket_connect_timeout")?, - socket_keepalive: optional_bool(&resolved, "socket_keepalive")?, - health_check_interval, - client_name: optional_dict_string(&resolved, "client_name")?, - tls, - }, + connection: resolved_connection(&resolved, host, port, pool_size, tls)?, })) } +/// The connection settings redis-py resolved for one client's pool. +#[inline(never)] +fn resolved_connection( + resolved: &Bound<'_, PyDict>, + host: String, + port: u16, + pool_size: usize, + tls: Option, +) -> PyResult { + let protocol = match optional_i64(resolved, "protocol")?.unwrap_or(2) { + 2 => RedisProtocol::Resp2, + 3 => RedisProtocol::Resp3, + _ => return Err(PyValueError::new_err("unsupported Redis protocol version")), + }; + Ok(RedisConnectionConfig { + host, + port, + database: optional_i64(resolved, "db")?.unwrap_or(0), + username: optional_dict_string(resolved, "username")?, + password: optional_dict_string(resolved, "password")?, + protocol, + pool_size, + read_timeout: optional_dict_duration(resolved, "socket_timeout")?, + connect_timeout: optional_dict_duration(resolved, "socket_connect_timeout")?, + socket_keepalive: optional_bool(resolved, "socket_keepalive")?, + health_check_interval: duration( + optional_f64(resolved, "health_check_interval")?.unwrap_or(0.0), + )?, + client_name: optional_dict_string(resolved, "client_name")?, + tls, + }) +} + #[inline(never)] fn project_s3( backend: &Bound<'_, PyAny>, @@ -933,31 +933,22 @@ fn project_valkey_semantic( } } if is_tls { + return Ok(Err(UnsupportedCacheConfig::ValkeyTls)); + } + let host = required_string(&resolved, "host")?; + if host.is_empty() { return Ok(Err(UnsupportedCacheConfig::RedisConnection)); } - let connection = RedisConnectionConfig { - host: required_string(&resolved, "host")?, - port: u16::try_from(required_i64(&resolved, "port")?) - .map_err(|_| PyValueError::new_err("invalid Redis port"))?, - database: optional_i64(&resolved, "db")?.unwrap_or(0), - username: optional_dict_string(&resolved, "username")?, - password: optional_dict_string(&resolved, "password")?, - protocol: RedisProtocol::Resp2, - pool_size: pool.getattr("max_connections")?.extract::()?, - read_timeout: None, - connect_timeout: None, - socket_keepalive: None, - health_check_interval: Duration::ZERO, - client_name: None, - tls: None, - }; - if connection.host.is_empty() { - return Ok(Err(UnsupportedCacheConfig::RedisConnection)); - } + let connection = resolved_connection( + &resolved, + host, + port(required_i64(&resolved, "port")?)?, + pool.getattr("max_connections")?.extract::()?, + None, + )?; Ok(Ok(ValkeySemanticCacheConfig { similarity_threshold: backend.getattr("similarity_threshold")?.extract()?, index_name: backend.getattr("index_name")?.extract()?, - embedding_model: backend.getattr("embedding_model")?.extract()?, connection, })) } @@ -1048,11 +1039,6 @@ fn class_is(value: &Bound<'_, PyAny>, module: &str, name: &str) -> PyResult) -> PyResult> { - value.extract::>()?.map(duration).transpose() -} - #[inline(never)] fn optional_attribute_string(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { match value.getattr(name) { @@ -1172,17 +1158,19 @@ fn optional_dict_duration(values: &Bound<'_, PyDict>, key: &str) -> PyResult(py: Python<'py>, startup_nodes: &str, hook: &str) -> Bound<'py, PyAny> { facade( @@ -1215,34 +1203,74 @@ mod tests { locals.get_item("facade").unwrap().unwrap() } - #[test] - fn projects_effective_memory_configuration() { + fn native(facade: &Bound<'_, PyAny>) -> NativeCacheConfig { + match NativeCacheConfig::project(facade).unwrap() { + CacheConfigProjection::Native(config) => *config, + CacheConfigProjection::Unsupported(reason) => panic!("{}", reason.message()), + } + } + + fn unsupported(facade: &Bound<'_, PyAny>) -> UnsupportedCacheConfig { + match NativeCacheConfig::project(facade).unwrap() { + CacheConfigProjection::Native(_) => panic!("configuration must stay on Python"), + CacheConfigProjection::Unsupported(reason) => reason, + } + } + + #[fixture] + fn interpreter() { Python::initialize(); + } + + #[fixture] + fn connection() -> RedisConnectionConfig { + RedisConnectionConfig { + host: "cache.internal".into(), + port: 6380, + database: 4, + username: None, + password: None, + protocol: RedisProtocol::Resp2, + pool_size: REDIS_PY_DEFAULT_MAX_CONNECTIONS, + read_timeout: None, + connect_timeout: None, + socket_keepalive: None, + health_check_interval: Duration::ZERO, + client_name: None, + tls: None, + } + } + + fn verified_tls() -> RedisTlsConfig { + RedisTlsConfig { + certificate_requirement: CertificateRequirement::Required, + check_hostname: true, + ca_certificate: None, + ca_data: None, + client_certificate: None, + client_key: None, + } + } + + #[rstest] + fn projects_effective_memory_configuration(_interpreter: ()) { Python::attach(|py| { let facade = facade( py, "backend = SimpleNamespace(default_ttl=913, max_size_in_memory=37, max_size_per_item=8)\n\ facade = SimpleNamespace(type='local', mode='default-on', ttl=11.5, namespace=None, supported_call_types=['completion'], redis_flush_size=None, semantic_cache_scope='key', cache=backend)", ); - let CacheConfigProjection::Native(config) = - NativeCacheConfig::project(&facade).unwrap() - else { - panic!("memory cache should be supported"); - }; - assert_eq!( - config.policy.ttl.unwrap(), - std::time::Duration::from_secs_f64(11.5) - ); + let config = native(&facade); + assert_eq!(config.policy.semantic_cache_scope, "key"); + assert_eq!(config.policy.redis_flush_size, None); let CacheBackendConfig::Memory(memory) = config.backend else { panic!("expected memory configuration"); }; - assert_eq!(memory.default_ttl, std::time::Duration::from_secs(913)); + assert_eq!(memory.default_ttl, Duration::from_secs(913)); assert_eq!(memory.capacity, 37); assert_eq!(memory.max_entry_bytes, 8192); - let matching = - NativeResponseCache::memory(37, std::time::Duration::from_secs(913), 8192); - let mismatched = - NativeResponseCache::memory(37, std::time::Duration::from_secs(913), 8191); + let matching = NativeResponseCache::memory(37, Duration::from_secs(913), 8192); + let mismatched = NativeResponseCache::memory(37, Duration::from_secs(913), 8191); let matching_config = NativeCacheConfig { policy: config.policy, backend: CacheBackendConfig::Memory(memory), @@ -1255,9 +1283,41 @@ mod tests { }); } - #[test] - fn projects_resolved_redis_tls_configuration() { - Python::initialize(); + #[rstest] + fn redis_semantic_service_mismatch_accepts_backend_precision_threshold(_interpreter: ()) { + Python::attach(|py| { + let facade = facade( + py, + "backend = SimpleNamespace(_redis_url='redis://127.0.0.1/', _index_name='semantic_idx', similarity_threshold=0.8, embedding_model='text-embedding-3-small', embedding_max_input_tokens=None, embedding_timeout=None)\n\ + facade = SimpleNamespace(type='redis-semantic', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", + ); + let backend = facade.getattr("cache").unwrap(); + let embedder = PythonEmbedder::new(backend.clone().unbind()); + let CacheBackendConfig::RedisSemantic(config) = native(&facade).backend else { + panic!("expected Redis semantic configuration"); + }; + let service = NativeResponseCache::redis_semantic( + &config.redis_url, + embedder, + RedisSemanticConfig { + index_name: config.index_name.clone(), + similarity_threshold: config.similarity_threshold as f32, + }, + ) + .unwrap(); + let matching_config = NativeCacheConfig { + policy: CachePolicy { + redis_flush_size: None, + semantic_cache_scope: "key".into(), + }, + backend: CacheBackendConfig::RedisSemantic(config), + }; + assert_eq!(matching_config.service_mismatch(&service), None); + }); + } + + #[rstest] + fn projects_resolved_redis_tls_configuration(_interpreter: ()) { Python::attach(|py| { let facade = facade( py, @@ -1269,15 +1329,12 @@ mod tests { backend = SimpleNamespace(default_ttl=777, namespace='team', redis_flush_size=31, redis_kwargs={}, redis_client=client)\n\ facade = SimpleNamespace(type='redis', mode='default-off', ttl=None, namespace='team', supported_call_types=None, redis_flush_size=31, semantic_cache_scope='key', cache=backend)", ); - let CacheConfigProjection::Native(config) = - NativeCacheConfig::project(&facade).unwrap() - else { - panic!("Redis cache should be supported"); - }; + let config = native(&facade); + assert_eq!(config.policy.redis_flush_size, Some(31)); let CacheBackendConfig::Redis(redis) = config.backend else { panic!("expected Redis configuration"); }; - assert_eq!(redis.default_ttl, std::time::Duration::from_secs(777)); + assert_eq!(redis.default_ttl, Duration::from_secs(777)); assert_eq!(redis.namespace.as_deref(), Some("team")); assert_eq!(redis.flush_size, 31); assert_eq!(redis.connection.host, "cache.internal"); @@ -1285,7 +1342,21 @@ mod tests { assert_eq!(redis.connection.database, 4); assert_eq!(redis.connection.protocol, RedisProtocol::Resp3); assert_eq!(redis.connection.pool_size, 29); - let tls = redis.connection.tls.unwrap(); + assert_eq!( + redis.connection.read_timeout, + Some(Duration::from_secs_f64(7.5)) + ); + assert_eq!( + redis.connection.connect_timeout, + Some(Duration::from_secs(2)) + ); + assert_eq!(redis.connection.socket_keepalive, Some(true)); + assert_eq!( + redis.connection.health_check_interval, + Duration::from_secs(15) + ); + assert_eq!(redis.connection.client_name.as_deref(), Some("litellm")); + let tls = redis.connection.tls.as_ref().unwrap(); assert_eq!( tls.certificate_requirement, CertificateRequirement::Optional @@ -1295,123 +1366,89 @@ mod tests { assert_eq!(tls.ca_data.as_deref(), Some("CA DATA")); assert_eq!(tls.client_certificate.as_deref(), Some("/client.pem")); assert_eq!(tls.client_key.as_deref(), Some("/client.key")); + assert!(matches!( + redis.connection.native_url(), + Err(UnsupportedCacheConfig::RedisPoolSize) + )); }); } - #[test] - fn projects_valkey_semantic_configuration() { - Python::initialize(); + #[rstest] + fn projects_valkey_semantic_configuration(_interpreter: ()) { Python::attach(|py| { let facade = facade( py, "pool = ConnectionPool()\n\ pool.connection_class = Connection\n\ pool.max_connections = 12\n\ - pool.connection_kwargs = {'host': 'cache.internal', 'port': 6390, 'db': 2}\n\ + pool.connection_kwargs = {'host': 'cache.internal', 'port': 6390, 'db': 2, 'socket_timeout': 3}\n\ client = SimpleNamespace(connection_pool=pool)\n\ backend = SimpleNamespace(similarity_threshold=0.85, index_name='semantic_idx', embedding_model='text-embedding-3-small', sync_client=client)\n\ facade = SimpleNamespace(type='valkey-semantic', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", ); - let CacheConfigProjection::Native(config) = - NativeCacheConfig::project(&facade).unwrap() - else { - panic!("Valkey semantic cache should be supported"); - }; - let CacheBackendConfig::ValkeySemantic(valkey) = config.backend else { + let CacheBackendConfig::ValkeySemantic(valkey) = native(&facade).backend else { panic!("expected Valkey semantic configuration"); }; assert_eq!(valkey.similarity_threshold, 0.85); assert_eq!(valkey.index_name, "semantic_idx"); - assert_eq!(valkey.embedding_model, "text-embedding-3-small"); assert_eq!(valkey.connection.host, "cache.internal"); assert_eq!(valkey.connection.port, 6390); assert_eq!(valkey.connection.database, 2); assert_eq!(valkey.connection.pool_size, 12); assert_eq!(valkey.connection.protocol, RedisProtocol::Resp2); + assert_eq!(valkey.connection.read_timeout, Some(Duration::from_secs(3))); assert!(valkey.connection.tls.is_none()); }); } - #[test] - fn valkey_semantic_tls_stays_on_python() { - Python::initialize(); + #[rstest] + #[case::valkey_tls( + "pool = ConnectionPool()\n\ + pool.connection_class = SSLConnection\n\ + pool.connection_kwargs = {'host': 'cache.internal', 'port': 6390}\n\ + client = SimpleNamespace(connection_pool=pool)\n\ + backend = SimpleNamespace(similarity_threshold=0.85, index_name='semantic_idx', embedding_model='text-embedding-3-small', sync_client=client)\n\ + facade = SimpleNamespace(type='valkey-semantic', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", + "native Valkey semantic cache does not support TLS connections" + )] + #[case::valkey_dynamic_auth( + "pool = ConnectionPool()\n\ + pool.connection_class = Connection\n\ + pool.connection_kwargs = {'host': 'cache.internal', 'port': 6390, 'credential_provider': object()}\n\ + client = SimpleNamespace(connection_pool=pool)\n\ + backend = SimpleNamespace(similarity_threshold=0.85, index_name='semantic_idx', embedding_model='text-embedding-3-small', sync_client=client)\n\ + facade = SimpleNamespace(type='valkey-semantic', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", + "native Redis credentials require Python" + )] + #[case::redis_dynamic_auth( + "backend = SimpleNamespace(redis_kwargs={'credential_provider': object()})\n\ + facade = SimpleNamespace(type='redis', mode='default-on', ttl=None, namespace=None, supported_call_types=[], redis_flush_size=None, semantic_cache_scope='key', cache=backend)", + "native Redis credentials require Python" + )] + #[case::gcs_without_bucket( + "backend = SimpleNamespace(bucket_name=None, key_prefix='', path_service_account=None)\n\ + facade = SimpleNamespace(type='gcs', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", + "native GCS cache requires a configured bucket name" + )] + fn configurations_that_stay_on_python( + _interpreter: (), + #[case] body: &str, + #[case] message: &str, + ) { Python::attach(|py| { - let facade = facade( - py, - "pool = ConnectionPool()\n\ - pool.connection_class = SSLConnection\n\ - pool.connection_kwargs = {'host': 'cache.internal', 'port': 6390}\n\ - client = SimpleNamespace(connection_pool=pool)\n\ - backend = SimpleNamespace(similarity_threshold=0.85, index_name='semantic_idx', embedding_model='text-embedding-3-small', sync_client=client)\n\ - facade = SimpleNamespace(type='valkey-semantic', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", - ); - let CacheConfigProjection::Unsupported(reason) = - NativeCacheConfig::project(&facade).unwrap() - else { - panic!("TLS Valkey semantic cache should stay on Python"); - }; - assert_eq!( - reason.message(), - "native Redis connection type is not implemented" - ); + assert_eq!(unsupported(&facade(py, body)).message(), message); }); } - #[test] - fn valkey_semantic_dynamic_auth_stays_on_python() { - Python::initialize(); - Python::attach(|py| { - let facade = facade( - py, - "pool = ConnectionPool()\n\ - pool.connection_class = Connection\n\ - pool.connection_kwargs = {'host': 'cache.internal', 'port': 6390, 'credential_provider': object()}\n\ - client = SimpleNamespace(connection_pool=pool)\n\ - backend = SimpleNamespace(similarity_threshold=0.85, index_name='semantic_idx', embedding_model='text-embedding-3-small', sync_client=client)\n\ - facade = SimpleNamespace(type='valkey-semantic', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", - ); - let CacheConfigProjection::Unsupported(reason) = - NativeCacheConfig::project(&facade).unwrap() - else { - panic!("dynamic Valkey authentication must stay on Python"); - }; - assert_eq!(reason.message(), "native Redis credentials require Python"); - }); - } - - #[test] - fn dynamic_redis_auth_stays_on_python() { - Python::initialize(); - Python::attach(|py| { - let facade = facade( - py, - "backend = SimpleNamespace(redis_kwargs={'credential_provider': object()})\n\ - facade = SimpleNamespace(type='redis', mode='default-on', ttl=None, namespace=None, supported_call_types=[], redis_flush_size=None, semantic_cache_scope='key', cache=backend)", - ); - let CacheConfigProjection::Unsupported(reason) = - NativeCacheConfig::project(&facade).unwrap() - else { - panic!("dynamic authentication must stay on Python"); - }; - assert_eq!(reason.message(), "native Redis credentials require Python"); - }); - } - - #[test] - fn projects_cluster_startup_nodes_as_redis_topology() { - Python::initialize(); + #[rstest] + fn projects_cluster_startup_nodes_as_redis_topology(_interpreter: ()) { Python::attach(|py| { let facade = cluster_facade( py, "[{'host': 'node-a', 'port': 7000}, {'host': 'node-b', 'port': 7001}]", "client.on_connect", ); - let CacheConfigProjection::Native(config) = - NativeCacheConfig::project(&facade).unwrap() - else { - panic!("cluster startup nodes should project natively"); - }; - let CacheBackendConfig::Redis(redis) = &config.backend else { + let CacheBackendConfig::Redis(redis) = native(&facade).backend else { panic!("expected Redis configuration"); }; let expected = RedisTopology::Cluster { @@ -1440,23 +1477,22 @@ mod tests { .certificate_requirement, CertificateRequirement::None ); + assert!(matches!( + redis.connection.native_url(), + Err(UnsupportedCacheConfig::RedisTlsVerification) + )); }); } - #[test] - fn projects_gcs_configuration() { - Python::initialize(); + #[rstest] + fn projects_gcs_configuration(_interpreter: ()) { Python::attach(|py| { let facade = facade( py, "backend = SimpleNamespace(bucket_name='bucket', key_prefix='cache/', path_service_account='credentials.json')\n\ facade = SimpleNamespace(type='gcs', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", ); - let CacheConfigProjection::Native(config) = - NativeCacheConfig::project(&facade).unwrap() - else { - panic!("GCS cache should be supported"); - }; + let config = native(&facade); let CacheBackendConfig::Gcs(gcs) = config.backend else { panic!("expected GCS configuration"); }; @@ -1475,9 +1511,9 @@ mod tests { path_service_account: Some("credentials.json".into()), endpoint: litellm_cache_gcs::DEFAULT_ENDPOINT.into(), }, + reqwest::Client::new(), Some("token".into()), - ) - .unwrap(); + ); let matching_config = NativeCacheConfig { policy: config.policy, backend: CacheBackendConfig::Gcs(gcs), @@ -1486,62 +1522,194 @@ mod tests { }); } - #[test] - fn rejects_gcs_without_a_bucket_name() { - Python::initialize(); + #[rstest] + #[case::extra_node_field( + "[{'host': 'node-a', 'port': 7000, 'server_type': 'primary'}]", + "client.on_connect", + "native Redis topology is not implemented" + )] + #[case::non_numeric_port( + "[{'host': 'node-a', 'port': 'seven'}]", + "client.on_connect", + "native Redis topology is not implemented" + )] + #[case::empty("[]", "client.on_connect", "native Redis topology is not implemented")] + #[case::foreign_hook( + "[{'host': 'node-a', 'port': 7000}]", + "lambda connection: None", + "native Redis credentials require Python" + )] + fn malformed_startup_nodes_and_foreign_connect_hooks_stay_on_python( + _interpreter: (), + #[case] startup_nodes: &str, + #[case] hook: &str, + #[case] message: &str, + ) { Python::attach(|py| { - let facade = facade( - py, - "backend = SimpleNamespace(bucket_name=None, key_prefix='', path_service_account=None)\n\ - facade = SimpleNamespace(type='gcs', mode='default-on', ttl=None, namespace=None, supported_call_types=None, redis_flush_size=None, semantic_cache_scope='key', cache=backend)", - ); - let CacheConfigProjection::Unsupported(reason) = - NativeCacheConfig::project(&facade).unwrap() - else { - panic!("GCS cache without a bucket should be unsupported"); - }; - assert!(matches!(&reason, UnsupportedCacheConfig::GcsBucket)); - assert_eq!( - reason.message(), - "native GCS cache requires a configured bucket name" - ); + let facade = cluster_facade(py, startup_nodes, hook); + assert_eq!(unsupported(&facade).message(), message); }); } - #[test] - fn malformed_startup_nodes_and_foreign_connect_hooks_stay_on_python() { - Python::initialize(); - Python::attach(|py| { - for (startup_nodes, hook, message) in [ - ( - "[{'host': 'node-a', 'port': 7000, 'server_type': 'primary'}]", - "client.on_connect", - "native Redis topology is not implemented", - ), - ( - "[{'host': 'node-a', 'port': 'seven'}]", - "client.on_connect", - "native Redis topology is not implemented", - ), - ( - "[]", - "client.on_connect", - "native Redis topology is not implemented", - ), - ( - "[{'host': 'node-a', 'port': 7000}]", - "lambda connection: None", - "native Redis credentials require Python", - ), - ] { - let facade = cluster_facade(py, startup_nodes, hook); - let CacheConfigProjection::Unsupported(reason) = - NativeCacheConfig::project(&facade).unwrap() - else { - panic!("{startup_nodes} with {hook} must stay on Python"); - }; - assert_eq!(reason.message(), message, "{startup_nodes} with {hook}"); + #[rstest] + #[case::plain(|_: &mut RedisConnectionConfig| {}, "redis://cache.internal:6380/4")] + #[case::credentials( + |connection: &mut RedisConnectionConfig| { + connection.username = Some("user".into()); + connection.password = Some("p@ss:word".into()); + }, + "redis://user:p%40ss%3Aword@cache.internal:6380/4" + )] + #[case::password_only( + |connection: &mut RedisConnectionConfig| connection.password = Some("secret".into()), + "redis://:secret@cache.internal:6380/4" + )] + #[case::resp3( + |connection: &mut RedisConnectionConfig| connection.protocol = RedisProtocol::Resp3, + "redis://cache.internal:6380/4?protocol=resp3" + )] + #[case::ipv6( + |connection: &mut RedisConnectionConfig| connection.host = "::1".into(), + "redis://[::1]:6380/4" + )] + #[case::verified_tls( + |connection: &mut RedisConnectionConfig| connection.tls = Some(verified_tls()), + "rediss://cache.internal:6380/4" + )] + #[case::optional_certificate( + |connection: &mut RedisConnectionConfig| { + connection.tls = Some(RedisTlsConfig { + certificate_requirement: CertificateRequirement::Optional, + ..verified_tls() + }); + }, + "rediss://cache.internal:6380/4" + )] + #[case::keepalive_off( + |connection: &mut RedisConnectionConfig| connection.socket_keepalive = Some(false), + "redis://cache.internal:6380/4" + )] + #[case::redis_cache_socket_timeout( + |connection: &mut RedisConnectionConfig| { + connection.read_timeout = Some(Duration::from_secs(5)); + }, + "redis://cache.internal:6380/4" + )] + fn native_url_encodes_the_resolved_connection( + mut connection: RedisConnectionConfig, + #[case] configure: fn(&mut RedisConnectionConfig), + #[case] expected: &str, + ) { + configure(&mut connection); + assert_eq!(connection.native_url().ok().as_deref(), Some(expected)); + } + + #[rstest] + #[case::pool_size( + |connection: &mut RedisConnectionConfig| connection.pool_size = 50, + "native Redis uses a fixed connection pool; max_connections requires Python" + )] + #[case::socket_timeout( + |connection: &mut RedisConnectionConfig| { + connection.read_timeout = Some(Duration::from_millis(100)); + }, + "native Redis uses fixed socket timeouts; socket_timeout and socket_connect_timeout require Python" + )] + #[case::connect_timeout( + |connection: &mut RedisConnectionConfig| { + connection.connect_timeout = Some(Duration::from_secs(1)); + }, + "native Redis uses fixed socket timeouts; socket_timeout and socket_connect_timeout require Python" + )] + #[case::keepalive( + |connection: &mut RedisConnectionConfig| connection.socket_keepalive = Some(true), + "native Redis does not support socket_keepalive" + )] + #[case::health_check( + |connection: &mut RedisConnectionConfig| { + connection.health_check_interval = Duration::from_secs(25); + }, + "native Redis does not support health_check_interval" + )] + #[case::client_name( + |connection: &mut RedisConnectionConfig| connection.client_name = Some("litellm".into()), + "native Redis does not support client_name" + )] + #[case::custom_ca( + |connection: &mut RedisConnectionConfig| { + connection.tls = Some(RedisTlsConfig { + ca_certificate: Some("/ca.pem".into()), + ..verified_tls() + }); + }, + "native Redis TLS does not support ssl_ca_certs, ssl_ca_data, ssl_certfile or ssl_keyfile" + )] + #[case::client_certificate( + |connection: &mut RedisConnectionConfig| { + connection.tls = Some(RedisTlsConfig { + client_certificate: Some("/client.pem".into()), + client_key: Some("/client.key".into()), + ..verified_tls() + }); + }, + "native Redis TLS does not support ssl_ca_certs, ssl_ca_data, ssl_certfile or ssl_keyfile" + )] + #[case::unverified( + |connection: &mut RedisConnectionConfig| { + connection.tls = Some(RedisTlsConfig { + certificate_requirement: CertificateRequirement::None, + check_hostname: false, + ..verified_tls() + }); + }, + "native Redis TLS always verifies the certificate and hostname; ssl_cert_reqs=none and ssl_check_hostname=false require Python" + )] + #[case::hostname_unchecked( + |connection: &mut RedisConnectionConfig| { + connection.tls = Some(RedisTlsConfig { + check_hostname: false, + ..verified_tls() + }); + }, + "native Redis TLS always verifies the certificate and hostname; ssl_cert_reqs=none and ssl_check_hostname=false require Python" + )] + fn native_url_declines_settings_the_native_client_cannot_honor( + mut connection: RedisConnectionConfig, + #[case] configure: fn(&mut RedisConnectionConfig), + #[case] message: &str, + ) { + configure(&mut connection); + let Err(reason) = connection.native_url() else { + panic!("{message}"); + }; + assert_eq!(reason.message(), message); + } + + #[rstest] + #[case::plain("redis://:secret@127.0.0.1:6379", true)] + #[case::database("redis://127.0.0.1:6379/2", true)] + #[case::unix("unix:///tmp/redis.sock", true)] + #[case::tls("rediss://cache.internal:6380", false)] + #[case::query_options("redis://127.0.0.1:6379?socket_timeout=1", false)] + #[case::malformed("not a url", false)] + fn redis_semantic_native_url_accepts_only_plain_urls(#[case] url: &str, #[case] native: bool) { + let config = RedisSemanticCacheConfig { + redis_url: url.into(), + index_name: "idx".into(), + similarity_threshold: 0.8, + }; + match config.native_url() { + Ok(value) => { + assert!(native, "{url} must decline"); + assert_eq!(value, url); } - }); + Err(reason) => { + assert!(!native, "{url} must be native"); + assert_eq!( + reason.message(), + "native Redis semantic cache does not support TLS or query options in redis_url" + ); + } + } } } diff --git a/litellm-rust/crates/python-bridge/src/cache/embedder.rs b/litellm-rust/crates/python-bridge/src/cache/embedder.rs index 9398e5a862b..7eadd9bc4b0 100644 --- a/litellm-rust/crates/python-bridge/src/cache/embedder.rs +++ b/litellm-rust/crates/python-bridge/src/cache/embedder.rs @@ -9,6 +9,8 @@ tokio::task_local! { static PREPARED_EMBEDDING: Result, Error>; } +/// Runs `future` with the vector the Python embedder already produced, so the backend's +/// `async_embed` never has to call back into Python from the runtime. pub(super) fn with_prepared_embedding( vector: Result, Error>, future: F, @@ -16,6 +18,7 @@ pub(super) fn with_prepared_embedding( PREPARED_EMBEDDING.scope(vector, future) } +/// The Python object that owns embedding for a semantic backend. pub(super) struct PythonEmbedder(Py); impl Clone for PythonEmbedder { @@ -29,10 +32,6 @@ impl PythonEmbedder { Self(object) } - pub(super) fn from_backend(backend: &Bound<'_, PyAny>) -> PyResult { - Ok(Self(backend.clone().unbind())) - } - pub(super) fn object(&self) -> &Py { &self.0 } @@ -41,18 +40,6 @@ impl PythonEmbedder { visit.call(&self.0) } - pub(super) fn async_embed_awaitable<'py>( - &self, - py: Python<'py>, - prompt: &str, - metadata: &Option, - ) -> PyResult> { - let metadata = to_py(py, metadata)?; - self.0 - .bind(py) - .call_method1("_get_async_embedding", (prompt, metadata)) - } - fn metadata_kwargs<'py>( py: Python<'py>, metadata: Option<&Value>, @@ -62,7 +49,8 @@ impl PythonEmbedder { Ok(kwargs) } - pub(super) fn async_embedding_coroutine( + /// The awaitable of `_get_async_embedding(prompt, metadata=...)`, to run in the caller's loop. + pub(super) fn async_embedding( &self, py: Python<'_>, prompt: &str, @@ -82,35 +70,8 @@ impl PythonEmbedder { .map(|value| value as f32) .collect()) } -} -impl litellm_cache_valkey_semantic::Embedder for PythonEmbedder { - fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { - let result = Python::attach(|py| -> PyResult> { - let metadata = to_py(py, &metadata)?; - self.0 - .bind(py) - .call_method1("_get_embedding", (prompt, metadata))? - .extract() - }) - .map_err(|_| Error::Unavailable)?; - Ok(result.into_iter().map(|value| value as f32).collect()) - } - - fn async_embed( - &self, - _prompt: &str, - _metadata: Option<&Value>, - ) -> impl Future, Error>> + Send { - let seeded = PREPARED_EMBEDDING - .try_with(Clone::clone) - .unwrap_or(Err(Error::Unavailable)); - std::future::ready(seeded) - } -} - -impl litellm_cache_redis_semantic::Embedder for PythonEmbedder { - fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { + fn embed_sync(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { Python::attach(|py| { let kwargs = Self::metadata_kwargs(py, metadata)?; Self::extract(self.0.bind(py).call_method( @@ -122,15 +83,24 @@ impl litellm_cache_redis_semantic::Embedder for PythonEmbedder { .map_err(|_| Error::Unavailable) } + fn seeded_embedding() -> Result, Error> { + PREPARED_EMBEDDING + .try_with(Clone::clone) + .unwrap_or(Err(Error::Unavailable)) + } +} + +impl litellm_cache::semantic::Embedder for PythonEmbedder { + fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result, Error> { + self.embed_sync(prompt, metadata) + } + fn async_embed( &self, _prompt: &str, _metadata: Option<&Value>, ) -> impl Future, Error>> + Send { - let seeded = PREPARED_EMBEDDING - .try_with(Clone::clone) - .unwrap_or(Err(Error::Unavailable)); - std::future::ready(seeded) + std::future::ready(Self::seeded_embedding()) } } @@ -145,12 +115,15 @@ mod tests { let embedder = PythonEmbedder::new(object); let scoped_embedder = embedder.clone(); let scoped = with_prepared_embedding(Ok(vec![0.25]), async move { - litellm_cache_redis_semantic::Embedder::async_embed(&scoped_embedder, "prompt", None) - .await + litellm_cache::semantic::Embedder::async_embed(&scoped_embedder, "prompt", None).await }); assert_eq!(scoped.await, Ok(vec![0.25])); let unscoped = - litellm_cache_redis_semantic::Embedder::async_embed(&embedder, "prompt", None).await; + litellm_cache::semantic::Embedder::async_embed(&embedder, "prompt", None).await; assert_eq!(unscoped, Err(Error::Unavailable)); + let valkey = with_prepared_embedding(Ok(vec![0.5]), async move { + litellm_cache::semantic::Embedder::async_embed(&embedder, "prompt", None).await + }); + assert_eq!(valkey.await, Ok(vec![0.5])); } } diff --git a/litellm-rust/crates/python-bridge/src/cache/facade.rs b/litellm-rust/crates/python-bridge/src/cache/facade.rs index 43b53584943..88fde2f6de0 100644 --- a/litellm-rust/crates/python-bridge/src/cache/facade.rs +++ b/litellm-rust/crates/python-bridge/src/cache/facade.rs @@ -11,6 +11,7 @@ use serde_json::Value; use super::{ config::{CacheConfigProjection, NativeCacheConfig}, handle::CacheTestHandle, + identity::BackendIdentity, native::NativeResponseCache, }; @@ -79,6 +80,10 @@ const CLUSTER_POOL: RedisPoolAttributes = RedisPoolAttributes { const VALKEY_POOL: RedisPoolAttributes = STANDALONE_POOL; +/// Class-level defaults an instance overwrites with its own state rather than behavior: +/// `Cache._native_cache` holds the runtime `Cache.__init__` resolved. +const INSTANCE_STATE: &[&str] = &["_native_cache"]; + pub(super) struct FacadeGuard { outer: ObjectGuard, backend: ObjectGuard, @@ -165,7 +170,9 @@ impl ObjectGuard { return Ok(false); } for (name, value) in &expected.attributes { - if (instance.contains(name)? && !self.config_names.contains(&name.as_str())) + if (instance.contains(name)? + && !self.config_names.contains(&name.as_str()) + && !INSTANCE_STATE.contains(&name.as_str())) || !attributes.get_item(name)?.is(value.bind(py)) { return Ok(false); @@ -352,47 +359,41 @@ impl FacadeGuard { facade: &Bound<'_, PyAny>, service: &NativeResponseCache, ) -> PyResult { - let kind = service.kind(); + let identity = service.identity(); + let kind = identity.kind(); let cache_type = py.import("litellm.caching.caching")?.getattr("Cache")?; if !facade.get_type().is(&cache_type) { return Err(PyTypeError::new_err( "only exact built-in Cache facades can be registered", )); } - let cluster = matches!(service.topology(), Some(RedisTopology::Cluster { .. })); - let (module, name, cache_kind) = match (kind, cluster) { - ("memory", _) => ("litellm.caching.in_memory_cache", "InMemoryCache", "local"), - ("redis", false) => ("litellm.caching.redis_cache", "RedisCache", "redis"), - ("redis_semantic", _) => ( - "litellm.caching.redis_semantic_cache", - "RedisSemanticCache", - "redis-semantic", - ), + let cluster = matches!( + identity, + BackendIdentity::Redis { + topology: RedisTopology::Cluster { .. }, + .. + } + ); + let (module, name) = match (kind, cluster) { + ("memory", _) => ("litellm.caching.in_memory_cache", "InMemoryCache"), + ("redis", false) => ("litellm.caching.redis_cache", "RedisCache"), + ("redis", true) => ("litellm.caching.redis_cluster_cache", "RedisClusterCache"), + ("redis_semantic", _) => ("litellm.caching.redis_semantic_cache", "RedisSemanticCache"), ("qdrant_semantic", _) => ( "litellm.caching.qdrant_semantic_cache", "QdrantSemanticCache", - "qdrant-semantic", ), - ("redis", true) => ( - "litellm.caching.redis_cluster_cache", - "RedisClusterCache", - "redis", - ), - ("gcs", _) => ("litellm.caching.gcs_cache", "GCSCache", "gcs"), - ("valkey-semantic", false) => ( + ("gcs", _) => ("litellm.caching.gcs_cache", "GCSCache"), + ("valkey-semantic", _) => ( "litellm.caching.valkey_semantic_cache", "ValkeySemanticCache", - "valkey-semantic", ), - ("disk", _) => ("litellm.caching.disk_cache", "DiskCache", "disk"), - ("azure-blob", _) => ( - "litellm.caching.azure_blob_cache", - "AzureBlobCache", - "azure-blob", - ), - ("s3", _) => ("litellm.caching.s3_cache", "S3Cache", "s3"), + ("disk", _) => ("litellm.caching.disk_cache", "DiskCache"), + ("azure-blob", _) => ("litellm.caching.azure_blob_cache", "AzureBlobCache"), + ("s3", _) => ("litellm.caching.s3_cache", "S3Cache"), _ => unreachable!(), }; + let cache_kind = identity.cache_type(); let backend = facade.getattr("cache")?; if facade.getattr("type")?.extract::()? != cache_kind || !backend.get_type().is(&py.import(module)?.getattr(name)?) diff --git a/litellm-rust/crates/python-bridge/src/cache/handle.rs b/litellm-rust/crates/python-bridge/src/cache/handle.rs index 51e1b02c405..dff8a771a23 100644 --- a/litellm-rust/crates/python-bridge/src/cache/handle.rs +++ b/litellm-rust/crates/python-bridge/src/cache/handle.rs @@ -10,7 +10,6 @@ use pyo3::{ PyTraverseError, PyVisit, exceptions::{PyRuntimeError, PyTypeError}, prelude::*, - types::PyDict, }; use url::Url; @@ -19,6 +18,7 @@ use super::{ config::{QdrantSemanticCacheConfig, project_redis_semantic}, embedder::PythonEmbedder, facade::FacadeGuard, + host_client, native::NativeResponseCache, request::duration, }; @@ -109,7 +109,10 @@ impl CacheTestHandle { ..Default::default() }, }; - let service = run_sync_value(py, async move { Ok(NativeResponseCache::s3(config).await) })?; + let http = host_client(py, ClientVariant::NoRedirect)?; + let service = run_sync_value(py, async move { + Ok(NativeResponseCache::s3(config, http).await) + })?; Ok(Self { service, guard: None, @@ -133,8 +136,8 @@ impl CacheTestHandle { path_service_account, endpoint: endpoint.unwrap_or_else(|| DEFAULT_ENDPOINT.to_string()), }; - let service = release_gil(py, move || NativeResponseCache::gcs(config, token)) - .map_err(cache_error)?; + let client = host_client(py, ClientVariant::NoRedirect)?; + let service = NativeResponseCache::gcs(config, client, token); Ok(Self { service, guard: None, @@ -236,10 +239,7 @@ impl CacheTestHandle { }, quantization, }; - let http_config = crate::http::call_config(py, &PyDict::new(py), true)?; - let client = crate::http::pool() - .client(&http_config, ClientVariant::Provider) - .map_err(crate::http::client_error)?; + let client = host_client(py, ClientVariant::Provider)?; let service = run_sync_value(py, async move { let handle = tokio::runtime::Handle::current(); NativeResponseCache::qdrant_semantic(config, client, handle) @@ -261,7 +261,7 @@ impl CacheTestHandle { index_name: String, embedder: &Bound<'_, PyAny>, ) -> PyResult { - let python_embedder = PythonEmbedder::from_backend(embedder)?; + let python_embedder = PythonEmbedder::new(embedder.clone().unbind()); let service = NativeResponseCache::valkey_semantic( &url, similarity_threshold, @@ -279,8 +279,9 @@ impl CacheTestHandle { #[staticmethod] #[pyo3(signature = (account_url, container))] fn azure_blob(py: Python<'_>, account_url: String, container: String) -> PyResult { + let http = host_client(py, ClientVariant::NoRedirect)?; let service = run_sync_value(py, async move { - NativeResponseCache::azure_blob(&account_url, &container) + NativeResponseCache::azure_blob(&account_url, &container, http) .await .map_err(cache_error) })?; diff --git a/litellm-rust/crates/python-bridge/src/cache/identity.rs b/litellm-rust/crates/python-bridge/src/cache/identity.rs new file mode 100644 index 00000000000..835bafd3ff1 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/cache/identity.rs @@ -0,0 +1,511 @@ +use std::{path::PathBuf, time::Duration}; + +use litellm_cache_redis::RedisTopology; + +/// What makes a native backend the one a Python facade describes: the configuration a user can +/// observe on the Python object, captured once so facade projection and native construction +/// compare plain data instead of reaching into each backend type. +#[derive(Clone, Debug, PartialEq)] +pub(super) enum BackendIdentity { + Memory { + capacity: usize, + max_entry_bytes: Option, + default_ttl: Option, + }, + Redis { + topology: RedisTopology, + namespace: Option, + default_ttl: Option, + }, + S3 { + bucket: String, + key_prefix: String, + region: String, + endpoint: Option, + }, + Gcs { + bucket_name: String, + key_prefix: String, + path_service_account: Option, + }, + Disk { + directory: PathBuf, + }, + AzureBlob { + account_url: String, + container: String, + }, + RedisSemantic { + index_name: String, + /// The backend stores the threshold as `f32`; a facade's `f64` is compared at that width. + similarity_threshold: f32, + }, + ValkeySemantic { + index_name: String, + similarity_threshold: f64, + }, + QdrantSemantic { + collection_name: String, + similarity_threshold: f64, + vector_size: u64, + embedding_model: String, + }, +} + +const TYPES: &str = "facade and native backend types must match"; + +impl BackendIdentity { + /// The native backend name reported to Python through `_CacheTestHandle.backend`. + pub(super) fn kind(&self) -> &'static str { + match self { + Self::Memory { .. } => "memory", + Self::Redis { .. } => "redis", + Self::S3 { .. } => "s3", + Self::Gcs { .. } => "gcs", + Self::ValkeySemantic { .. } => "valkey-semantic", + Self::RedisSemantic { .. } => "redis_semantic", + Self::QdrantSemantic { .. } => "qdrant_semantic", + Self::Disk { .. } => "disk", + Self::AzureBlob { .. } => "azure-blob", + } + } + + /// The `LiteLLMCacheType` value a facade of this backend carries in `Cache.type`. + pub(super) fn cache_type(&self) -> &'static str { + match self { + Self::Memory { .. } => "local", + Self::Redis { .. } => "redis", + Self::S3 { .. } => "s3", + Self::Gcs { .. } => "gcs", + Self::ValkeySemantic { .. } => "valkey-semantic", + Self::RedisSemantic { .. } => "redis-semantic", + Self::QdrantSemantic { .. } => "qdrant-semantic", + Self::Disk { .. } => "disk", + Self::AzureBlob { .. } => "azure-blob", + } + } + + /// The first difference between the facade's configuration (`self`) and the native + /// backend (`native`), in the order Python users see the attributes. + pub(super) fn mismatch(&self, native: &Self) -> Option<&'static str> { + let mut differences: Vec<(bool, &'static str)> = Vec::new(); + let mut differs = |condition: bool, message: &'static str| { + differences.push((condition, message)); + }; + match (self, native) { + ( + Self::Memory { + capacity, + max_entry_bytes, + default_ttl, + }, + Self::Memory { + capacity: native_capacity, + max_entry_bytes: native_max_entry_bytes, + default_ttl: native_default_ttl, + }, + ) => { + differs( + default_ttl != native_default_ttl, + "facade and native backend default TTLs must match", + ); + differs( + capacity != native_capacity, + "facade and native backend capacities must match", + ); + differs( + max_entry_bytes != native_max_entry_bytes, + "facade and native backend item limits must match", + ); + } + ( + Self::Redis { + topology, + namespace, + default_ttl, + }, + Self::Redis { + topology: native_topology, + namespace: native_namespace, + default_ttl: native_default_ttl, + }, + ) => { + differs( + default_ttl != native_default_ttl, + "facade and native backend default TTLs must match", + ); + differs( + topology != native_topology, + "facade and native backend topologies must match", + ); + differs( + namespace != native_namespace, + "facade and native backend namespaces must match", + ); + } + ( + Self::S3 { + bucket, + key_prefix, + region, + endpoint, + }, + Self::S3 { + bucket: native_bucket, + key_prefix: native_key_prefix, + region: native_region, + endpoint: native_endpoint, + }, + ) => { + differs( + bucket != native_bucket, + "facade and native backend buckets must match", + ); + differs( + key_prefix != native_key_prefix, + "facade and native backend key prefixes must match", + ); + differs( + region != native_region, + "facade and native backend regions must match", + ); + differs( + endpoint != native_endpoint, + "facade and native backend endpoints must match", + ); + } + ( + Self::Gcs { + bucket_name, + key_prefix, + path_service_account, + }, + Self::Gcs { + bucket_name: native_bucket_name, + key_prefix: native_key_prefix, + path_service_account: native_path_service_account, + }, + ) => { + differs( + bucket_name != native_bucket_name, + "facade and native backend buckets must match", + ); + differs( + key_prefix != native_key_prefix, + "facade and native backend key prefixes must match", + ); + differs( + path_service_account != native_path_service_account, + "facade and native backend credentials must match", + ); + } + ( + Self::Disk { directory }, + Self::Disk { + directory: native_directory, + }, + ) => { + let canonical = |path: &PathBuf| std::fs::canonicalize(path).ok(); + differs( + canonical(directory) != canonical(native_directory), + "facade and native backend directories must match", + ); + } + ( + Self::AzureBlob { + account_url, + container, + }, + Self::AzureBlob { + account_url: native_account_url, + container: native_container, + }, + ) => { + differs( + account_url != native_account_url || container != native_container, + "facade and native backend containers must match", + ); + } + ( + Self::RedisSemantic { + index_name, + similarity_threshold, + }, + Self::RedisSemantic { + index_name: native_index_name, + similarity_threshold: native_similarity_threshold, + }, + ) => { + differs( + index_name != native_index_name, + "facade and native backend index names must match", + ); + differs( + similarity_threshold != native_similarity_threshold, + "facade and native backend similarity thresholds must match", + ); + } + ( + Self::ValkeySemantic { + index_name, + similarity_threshold, + }, + Self::ValkeySemantic { + index_name: native_index_name, + similarity_threshold: native_similarity_threshold, + }, + ) => { + differs( + index_name != native_index_name + || similarity_threshold != native_similarity_threshold, + "facade and native semantic settings must match", + ); + } + ( + Self::QdrantSemantic { + collection_name, + similarity_threshold, + vector_size, + embedding_model, + }, + Self::QdrantSemantic { + collection_name: native_collection_name, + similarity_threshold: native_similarity_threshold, + vector_size: native_vector_size, + embedding_model: native_embedding_model, + }, + ) => { + differs( + collection_name != native_collection_name, + "facade and native backend collections must match", + ); + differs( + similarity_threshold != native_similarity_threshold, + "facade and native backend similarity thresholds must match", + ); + differs( + vector_size != native_vector_size, + "facade and native backend vector sizes must match", + ); + differs( + embedding_model != native_embedding_model, + "facade and native backend embedding models must match", + ); + } + _ => return Some(TYPES), + } + differences + .into_iter() + .find_map(|(condition, message)| condition.then_some(message)) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use litellm_cache_redis::{RedisNode, RedisTopology}; + + use super::BackendIdentity; + + fn memory() -> BackendIdentity { + BackendIdentity::Memory { + capacity: 200, + max_entry_bytes: Some(1024), + default_ttl: Some(Duration::from_secs(60)), + } + } + + fn redis() -> BackendIdentity { + BackendIdentity::Redis { + topology: RedisTopology::Standalone, + namespace: Some("team".into()), + default_ttl: Some(Duration::from_secs(60)), + } + } + + fn s3() -> BackendIdentity { + BackendIdentity::S3 { + bucket: "bucket".into(), + key_prefix: "cache/".into(), + region: "us-east-1".into(), + endpoint: None, + } + } + + fn gcs() -> BackendIdentity { + BackendIdentity::Gcs { + bucket_name: "bucket".into(), + key_prefix: "cache/".into(), + path_service_account: Some("credentials.json".into()), + } + } + + fn azure() -> BackendIdentity { + BackendIdentity::AzureBlob { + account_url: "https://account.blob.core.windows.net".into(), + container: "cache".into(), + } + } + + fn redis_semantic() -> BackendIdentity { + BackendIdentity::RedisSemantic { + index_name: "idx".into(), + similarity_threshold: 0.8, + } + } + + #[test] + fn redis_semantic_thresholds_compare_at_backend_precision() { + let facade = BackendIdentity::RedisSemantic { + index_name: "idx".into(), + similarity_threshold: 0.8_f64 as f32, + }; + assert_eq!(facade.mismatch(&redis_semantic()), None); + } + + fn valkey_semantic() -> BackendIdentity { + BackendIdentity::ValkeySemantic { + index_name: "idx".into(), + similarity_threshold: 0.8, + } + } + + fn qdrant() -> BackendIdentity { + BackendIdentity::QdrantSemantic { + collection_name: "collection".into(), + similarity_threshold: 0.8, + vector_size: 1536, + embedding_model: "text-embedding-3-small".into(), + } + } + + #[test] + fn identical_identities_have_no_mismatch() { + for identity in [ + memory(), + redis(), + s3(), + gcs(), + azure(), + redis_semantic(), + valkey_semantic(), + qdrant(), + BackendIdentity::Disk { + directory: std::env::temp_dir(), + }, + ] { + assert_eq!(identity.mismatch(&identity), None, "{identity:?}"); + } + } + + #[test] + fn different_kinds_report_a_type_mismatch() { + assert_eq!( + memory().mismatch(&redis()), + Some("facade and native backend types must match") + ); + assert_eq!( + redis_semantic().mismatch(&valkey_semantic()), + Some("facade and native backend types must match") + ); + } + + #[test] + fn the_first_differing_field_names_the_mismatch() { + let BackendIdentity::Memory { capacity, .. } = memory() else { + unreachable!() + }; + assert_eq!( + memory().mismatch(&BackendIdentity::Memory { + capacity: capacity + 1, + max_entry_bytes: Some(1), + default_ttl: Some(Duration::from_secs(60)), + }), + Some("facade and native backend capacities must match") + ); + assert_eq!( + memory().mismatch(&BackendIdentity::Memory { + capacity, + max_entry_bytes: Some(1), + default_ttl: Some(Duration::from_secs(61)), + }), + Some("facade and native backend default TTLs must match") + ); + assert_eq!( + redis().mismatch(&BackendIdentity::Redis { + topology: RedisTopology::Cluster { + startup_nodes: vec![RedisNode { + host: "node".into(), + port: 7000, + }], + }, + namespace: None, + default_ttl: Some(Duration::from_secs(60)), + }), + Some("facade and native backend topologies must match") + ); + assert_eq!( + s3().mismatch(&BackendIdentity::S3 { + bucket: "bucket".into(), + key_prefix: "cache/".into(), + region: "us-east-1".into(), + endpoint: Some("http://localhost:9000".into()), + }), + Some("facade and native backend endpoints must match") + ); + assert_eq!( + gcs().mismatch(&BackendIdentity::Gcs { + bucket_name: "bucket".into(), + key_prefix: "cache/".into(), + path_service_account: None, + }), + Some("facade and native backend credentials must match") + ); + assert_eq!( + azure().mismatch(&BackendIdentity::AzureBlob { + account_url: "https://account.blob.core.windows.net".into(), + container: "other".into(), + }), + Some("facade and native backend containers must match") + ); + assert_eq!( + valkey_semantic().mismatch(&BackendIdentity::ValkeySemantic { + index_name: "idx".into(), + similarity_threshold: 0.9, + }), + Some("facade and native semantic settings must match") + ); + assert_eq!( + qdrant().mismatch(&BackendIdentity::QdrantSemantic { + collection_name: "collection".into(), + similarity_threshold: 0.8, + vector_size: 1536, + embedding_model: "text-embedding-3-large".into(), + }), + Some("facade and native backend embedding models must match") + ); + } + + #[test] + fn disk_directories_compare_canonically() { + let directory = std::env::temp_dir(); + let mut indirect = directory.clone(); + indirect.push("."); + assert_eq!( + BackendIdentity::Disk { + directory: directory.clone() + } + .mismatch(&BackendIdentity::Disk { + directory: indirect + }), + None + ); + assert_eq!( + BackendIdentity::Disk { directory }.mismatch(&BackendIdentity::Disk { + directory: "/definitely/missing".into() + }), + Some("facade and native backend directories must match") + ); + } +} diff --git a/litellm-rust/crates/python-bridge/src/cache/mod.rs b/litellm-rust/crates/python-bridge/src/cache/mod.rs index fa028518559..0cfd4ac8138 100644 --- a/litellm-rust/crates/python-bridge/src/cache/mod.rs +++ b/litellm-rust/crates/python-bridge/src/cache/mod.rs @@ -1,3 +1,4 @@ +mod activation; mod binding; mod callback; mod config; @@ -5,16 +6,18 @@ mod embedder; mod facade; mod future; mod handle; +mod identity; mod native; mod request; mod resolver; mod semantic; -mod semantic_step; use litellm_cache::Error; +use litellm_http::ClientVariant; use pyo3::{ exceptions::{PyNotImplementedError, PyRuntimeError, PyValueError}, prelude::*, + types::PyDict, }; pub(crate) use self::{ @@ -28,3 +31,11 @@ fn cache_error(error: Error) -> PyErr { _ => PyRuntimeError::new_err(error.to_string()), } } + +/// The host's pooled HTTP client, configured from the proxy's HTTP settings. +fn host_client(py: Python<'_>, variant: ClientVariant) -> PyResult { + let http_config = crate::http::call_config(py, &PyDict::new(py), true)?; + crate::http::pool() + .client(&http_config, variant) + .map_err(crate::http::client_error) +} diff --git a/litellm-rust/crates/python-bridge/src/cache/native.rs b/litellm-rust/crates/python-bridge/src/cache/native.rs index 7ca62f10ec8..3c260e70843 100644 --- a/litellm-rust/crates/python-bridge/src/cache/native.rs +++ b/litellm-rust/crates/python-bridge/src/cache/native.rs @@ -1,18 +1,16 @@ -use std::{path::Path, sync::Arc, time::Duration}; +use std::{sync::Arc, time::Duration}; -use litellm_cache::{ - CacheCodec, CacheConnectionResult, Error, ExactCacheContext, SemanticCacheContext, -}; +use litellm_cache::{CacheCodec, CacheConnectionResult, Error, semantic::SemanticLookup}; use litellm_cache_azure_blob::AzureBlobCache; use litellm_cache_disk::DiskCache; use litellm_cache_gcs::{GcsCache, GcsConfig, StaticTokenSource}; use litellm_cache_memory::InMemoryCache; -use litellm_cache_qdrant_semantic::{Embedder, OpenAiEmbedder, QdrantSemanticCache}; +use litellm_cache_qdrant_semantic::{OpenAiEmbedder, QdrantSemanticCache}; use litellm_cache_redis::{RedisCache, RedisTopology}; use litellm_cache_redis_semantic::{RedisSemanticCache, RedisSemanticConfig}; use litellm_cache_response::{ - CacheEntry, CacheKeyField, PartialHits, ResponseCache, ResponseCacheCodec, - ResponseCacheRequest, WriteBuffer, + ConnectionProbe, ExactResponseCache, PartialHits, ResponseCache, ResponseCacheCodec, + WriteBuffer, }; use litellm_cache_s3::{S3Cache, S3CacheConfig}; use litellm_cache_valkey_semantic::{ValkeySemanticCache, ValkeySemanticConfig}; @@ -22,96 +20,57 @@ use serde_json::Value; use super::{ config::QdrantSemanticCacheConfig, embedder::PythonEmbedder, - request::NativeRequest, - semantic::{SemanticBody, SemanticOperation, drive}, - semantic_step::{SemanticEmbedExecution, drive_semantic}, + identity::BackendIdentity, + request::{NativeRequest, now}, + semantic::{EmbeddingFailure, SemanticExecution, SemanticOperation, drive}, }; -fn semantic_key(request: &NativeRequest, scope: &str) -> litellm_cache_response::CacheKeyInput { - let mut key = request.key.clone(); - if key.preset.is_some() { - return key; - } - key.fields - .retain(|field| !matches!(field.name.as_str(), "messages" | "prompt" | "input")); - const TENANT: [&str; 3] = [ - "user_api_key", - "user_api_key_team_id", - "user_api_key_org_id", - ]; - let end_user = (scope == "end_user").then_some("user_api_key_end_user_id"); - for name in TENANT.into_iter().chain(end_user) { - let sources = [ - request.metadata.as_ref(), - request.litellm_metadata.as_ref(), - request - .litellm_params - .as_ref() - .and_then(|params| params.get("metadata")), - request - .litellm_params - .as_ref() - .and_then(|params| params.get("litellm_metadata")), - ]; - let Some(value) = sources.into_iter().flatten().find_map(|source| { - source - .as_object() - .and_then(|values| values.get(name)) - .filter(|value| !value.is_null()) - }) else { - continue; - }; - let value = match value { - Value::Null => continue, - Value::String(text) => text.clone(), - other => other.to_string(), - }; - key.fields.push(CacheKeyField { - name: name.to_owned(), - value: Some(value), - api_parameter: true, - internal_parameter: false, - }); - } - key +/// What the Python embedder receives for one semantic request. +pub(super) struct EmbeddingInput { + pub(super) prompt: String, + pub(super) metadata: Option, +} + +/// An exact-match backend behind one pointer, with the identity its facade must reproduce. +pub(super) struct ExactService { + cache: Arc, + probe: Option>, + buffer: Option, + identity: BackendIdentity, } #[derive(Clone)] pub(super) enum NativeResponseCache { - Memory(Arc>>), - Redis { - cache: Arc>>, - buffer: Option>, - }, - S3(Arc>>), - Gcs(Arc>>), + Exact(Arc), ValkeySemantic { cache: Arc>>, embedder: PythonEmbedder, scope: String, }, RedisSemantic { - cache: Arc>>, + cache: Arc>>, embedder: PythonEmbedder, }, QdrantSemantic(Arc>>), - Disk(Arc>>), - AzureBlob(Arc>>), } impl NativeResponseCache { pub fn memory(capacity: usize, ttl: Duration, max_entry_bytes: usize) -> Self { - Self::Memory(Arc::new(ResponseCache::new(Arc::new( - InMemoryCache::with_clock_and_size_measurement( - Some(capacity), - Some(ttl), - Some(max_entry_bytes), - Some(Arc::new(|entry| { - ResponseCacheCodec.encode(entry).map(|bytes| bytes.len()) - })), - super::request::now, - ), - )))) + let backend = InMemoryCache::with_clock_and_size_measurement( + Some(capacity), + Some(ttl), + Some(max_entry_bytes), + Some(Arc::new(|entry| { + ResponseCacheCodec.encode(entry).map(|bytes| bytes.len()) + })), + now, + ); + let identity = BackendIdentity::Memory { + capacity: backend.max_size_in_memory(), + max_entry_bytes: backend.max_entry_bytes(), + default_ttl: None, + }; + Self::exact(ResponseCache::new(Arc::new(backend)), identity) } pub fn redis( @@ -122,19 +81,129 @@ impl NativeResponseCache { ) -> Result { let backend = RedisCache::connect(url, topology, ttl, ResponseCacheCodec)?.with_namespace(namespace); - Ok(Self::Redis { - cache: Arc::new(ResponseCache::new(Arc::new(backend))), - buffer: None, - }) + let identity = BackendIdentity::Redis { + topology: backend.topology().clone(), + namespace: backend.namespace().map(str::to_owned), + default_ttl: None, + }; + Ok(Self::exact_probed( + ResponseCache::new(Arc::new(backend)), + identity, + )) } - pub async fn s3(config: S3CacheConfig) -> Self { + pub async fn s3(config: S3CacheConfig, http: reqwest::Client) -> Self { let runtime = tokio::runtime::Handle::current(); - Self::S3(Arc::new(ResponseCache::new(Arc::new(S3Cache::new( - config, + let backend = S3Cache::new(config, http, ResponseCacheCodec, runtime); + let identity = BackendIdentity::S3 { + bucket: backend.bucket().to_owned(), + key_prefix: backend.key_prefix().to_owned(), + region: backend.region().to_owned(), + endpoint: backend.endpoint().map(str::to_owned), + }; + Self::exact(ResponseCache::new(Arc::new(backend)), identity) + } + + pub fn disk(directory: impl AsRef) -> Result { + let backend = DiskCache::open(directory, ResponseCacheCodec)?; + let identity = BackendIdentity::Disk { + directory: backend.directory().to_path_buf(), + }; + Ok(Self::exact(ResponseCache::new(Arc::new(backend)), identity)) + } + + pub fn gcs(config: GcsConfig, client: reqwest::Client, token: Option) -> Self { + let backend = match token { + Some(token) => GcsCache::with_token_source( + config, + client, + ResponseCacheCodec, + Arc::new(StaticTokenSource(token)), + ), + None => GcsCache::new(config, client, ResponseCacheCodec), + }; + let identity = BackendIdentity::Gcs { + bucket_name: backend.bucket_name().to_owned(), + key_prefix: backend.key_prefix().to_owned(), + path_service_account: backend.path_service_account().map(str::to_owned), + }; + Self::exact(ResponseCache::new(Arc::new(backend)), identity) + } + + pub async fn azure_blob( + account_url: &str, + container: &str, + http: reqwest::Client, + ) -> Result { + let backend = AzureBlobCache::connect( + account_url, + container, + http, ResponseCacheCodec, - runtime, - ))))) + tokio::runtime::Handle::current(), + ) + .await?; + let identity = BackendIdentity::AzureBlob { + account_url: backend.account_url().to_owned(), + container: backend.container_name().to_owned(), + }; + Ok(Self::exact(ResponseCache::new(Arc::new(backend)), identity)) + } + + /// Wraps a built exact backend; the TTL a facade must match comes from the built cache. + fn exact(cache: ResponseCache, identity: BackendIdentity) -> Self + where + ResponseCache: ExactResponseCache + 'static, + B: litellm_cache::BaseCache, + B::Context: Default + PartialEq, + { + Self::exact_service(Arc::new(cache), None, identity) + } + + /// Wraps an exact backend whose Python class defines `test_connection`. + fn exact_probed(cache: ResponseCache, identity: BackendIdentity) -> Self + where + ResponseCache: ExactResponseCache + ConnectionProbe + 'static, + B: litellm_cache::BaseCache, + B::Context: Default + PartialEq, + { + let cache = Arc::new(cache); + Self::exact_service(cache.clone(), Some(cache), identity) + } + + fn exact_service( + cache: Arc, + probe: Option>, + identity: BackendIdentity, + ) -> Self { + let default_ttl = cache.default_ttl(); + let identity = match identity { + BackendIdentity::Memory { + capacity, + max_entry_bytes, + .. + } => BackendIdentity::Memory { + capacity, + max_entry_bytes, + default_ttl, + }, + BackendIdentity::Redis { + topology, + namespace, + .. + } => BackendIdentity::Redis { + topology, + namespace, + default_ttl, + }, + other => other, + }; + Self::Exact(Arc::new(ExactService { + cache, + probe, + buffer: None, + identity, + })) } pub fn valkey_semantic( @@ -164,7 +233,7 @@ impl NativeResponseCache { embedder: PythonEmbedder, config: RedisSemanticConfig, ) -> Result { - let backend = RedisSemanticCache::new(url, embedder.clone(), config)?; + let backend = RedisSemanticCache::new(url, embedder.clone(), ResponseCacheCodec, config)?; Ok(Self::RedisSemantic { cache: Arc::new(ResponseCache::new(Arc::new(backend))), embedder, @@ -196,103 +265,40 @@ impl NativeResponseCache { )))) } - pub fn disk(directory: &str) -> Result { - let cache = DiskCache::open(directory, ResponseCacheCodec)?; - Ok(Self::Disk(Arc::new(ResponseCache::new(Arc::new(cache))))) - } - - pub fn gcs(config: GcsConfig, token: Option) -> Result { - let backend = match token { - Some(token) => GcsCache::with_token_source( - config, - ResponseCacheCodec, - Arc::new(StaticTokenSource(token)), - )?, - None => GcsCache::new(config, ResponseCacheCodec)?, - }; - Ok(Self::Gcs(Arc::new(ResponseCache::new(Arc::new(backend))))) - } - - pub async fn azure_blob(account_url: &str, container: &str) -> Result { - let backend = AzureBlobCache::connect( - account_url, - container, - ResponseCacheCodec, - tokio::runtime::Handle::current(), - ) - .await?; - Ok(Self::AzureBlob(Arc::new(ResponseCache::new(Arc::new( - backend, - ))))) - } - - pub fn azure_blob_identity(&self) -> Option<(&str, &str)> { + pub fn identity(&self) -> BackendIdentity { match self { - Self::AzureBlob(cache) => Some(( - cache.backend().account_url(), - cache.backend().container_name(), - )), - Self::Memory(_) - | Self::Redis { .. } - | Self::S3(_) - | Self::ValkeySemantic { .. } - | Self::RedisSemantic { .. } - | Self::QdrantSemantic(_) - | Self::Disk(_) - | Self::Gcs(_) => None, - } - } - - fn exact(request: &NativeRequest) -> ResponseCacheRequest { - ResponseCacheRequest { - key: request.key.clone(), - controls: request.controls, - context: ExactCacheContext { ttl: request.ttl }, - max_age: request.max_age, - } - } - - pub(super) fn semantic_request( - request: &NativeRequest, - ) -> ResponseCacheRequest { - ResponseCacheRequest { - key: request.key.clone(), - controls: request.controls, - context: SemanticCacheContext { - input: request.input.clone(), - messages: request.messages.clone(), - metadata: request.metadata.clone(), - scope: request.scope.clone(), - ttl: request.ttl, + Self::Exact(service) => service.identity.clone(), + Self::ValkeySemantic { cache, .. } => BackendIdentity::ValkeySemantic { + index_name: cache.backend().index_name().to_owned(), + similarity_threshold: cache.backend().similarity_threshold(), + }, + Self::RedisSemantic { cache, .. } => BackendIdentity::RedisSemantic { + index_name: cache.backend().index_name().to_owned(), + similarity_threshold: cache.backend().similarity_threshold(), + }, + Self::QdrantSemantic(cache) => BackendIdentity::QdrantSemantic { + collection_name: cache.backend().collection_name().to_owned(), + similarity_threshold: cache.backend().similarity_threshold(), + vector_size: cache.backend().vector_size(), + embedding_model: cache.backend().embedder().model().to_owned(), }, - max_age: request.max_age, } } - fn semantic( - request: &NativeRequest, - scope: &str, - ) -> ResponseCacheRequest { - ResponseCacheRequest { - key: semantic_key(request, scope), - controls: request.controls, - context: SemanticCacheContext { - input: request.input.clone(), - messages: request.messages.clone(), - metadata: request.metadata.clone(), - scope: Some(scope.to_owned()), - ttl: request.ttl, - }, - max_age: request.max_age, - } + pub fn kind(&self) -> &'static str { + self.identity().kind() } pub fn with_redis_flush_size(self, flush_size: Option) -> Self { match self { - Self::Redis { cache, .. } => Self::Redis { - cache, - buffer: flush_size.map(|size| Arc::new(WriteBuffer::new(size))), - }, + Self::Exact(service) if matches!(service.identity, BackendIdentity::Redis { .. }) => { + Self::Exact(Arc::new(ExactService { + cache: Arc::clone(&service.cache), + probe: service.probe.clone(), + buffer: flush_size.map(WriteBuffer::new), + identity: service.identity.clone(), + })) + } value => value, } } @@ -310,191 +316,6 @@ impl NativeResponseCache { } } - pub fn kind(&self) -> &'static str { - match self { - Self::Memory(_) => "memory", - Self::Redis { .. } => "redis", - Self::S3(_) => "s3", - Self::Gcs(_) => "gcs", - Self::ValkeySemantic { .. } => "valkey-semantic", - Self::RedisSemantic { .. } => "redis_semantic", - Self::QdrantSemantic(_) => "qdrant_semantic", - Self::Disk(_) => "disk", - Self::AzureBlob(_) => "azure-blob", - } - } - - pub fn default_ttl(&self) -> Option { - match self { - Self::Memory(cache) => cache.default_ttl(), - Self::Redis { cache, .. } => cache.default_ttl(), - Self::S3(cache) => cache.default_ttl(), - Self::Gcs(cache) => cache.default_ttl(), - Self::ValkeySemantic { cache, .. } => cache.default_ttl(), - Self::RedisSemantic { cache, .. } => cache.default_ttl(), - Self::QdrantSemantic(_) => None, - Self::Disk(cache) => cache.default_ttl(), - Self::AzureBlob(cache) => cache.default_ttl(), - } - } - - pub fn bucket(&self) -> Option<&str> { - match self { - Self::S3(cache) => Some(cache.backend().bucket()), - _ => None, - } - } - - pub fn key_prefix(&self) -> Option<&str> { - match self { - Self::S3(cache) => Some(cache.backend().key_prefix()), - _ => None, - } - } - - pub fn region(&self) -> Option<&str> { - match self { - Self::S3(cache) => Some(cache.backend().region()), - _ => None, - } - } - - pub fn endpoint(&self) -> Option<&str> { - match self { - Self::S3(cache) => cache.backend().endpoint(), - _ => None, - } - } - - pub fn namespace(&self) -> Option<&str> { - match self { - Self::Memory(_) - | Self::S3(_) - | Self::ValkeySemantic { .. } - | Self::RedisSemantic { .. } - | Self::QdrantSemantic(_) - | Self::Disk(_) - | Self::AzureBlob(_) - | Self::Gcs(_) => None, - Self::Redis { cache, .. } => cache.backend().namespace(), - } - } - - pub fn topology(&self) -> Option<&RedisTopology> { - match self { - Self::Memory(_) - | Self::S3(_) - | Self::ValkeySemantic { .. } - | Self::RedisSemantic { .. } - | Self::QdrantSemantic(_) - | Self::Disk(_) - | Self::AzureBlob(_) - | Self::Gcs(_) => None, - Self::Redis { cache, .. } => Some(cache.backend().topology()), - } - } - - pub fn capacity(&self) -> Option { - match self { - Self::Memory(cache) => Some(cache.backend().max_size_in_memory()), - Self::Redis { .. } - | Self::S3(_) - | Self::ValkeySemantic { .. } - | Self::RedisSemantic { .. } - | Self::QdrantSemantic(_) - | Self::Disk(_) - | Self::AzureBlob(_) - | Self::Gcs(_) => None, - } - } - - pub fn max_entry_bytes(&self) -> Option { - match self { - Self::Memory(cache) => cache.backend().max_entry_bytes(), - Self::Redis { .. } - | Self::S3(_) - | Self::ValkeySemantic { .. } - | Self::RedisSemantic { .. } - | Self::QdrantSemantic(_) - | Self::Disk(_) - | Self::AzureBlob(_) - | Self::Gcs(_) => None, - } - } - - pub fn directory(&self) -> Option<&Path> { - match self { - Self::Disk(cache) => Some(cache.backend().directory()), - Self::Memory(_) - | Self::Redis { .. } - | Self::S3(_) - | Self::ValkeySemantic { .. } - | Self::RedisSemantic { .. } - | Self::QdrantSemantic(_) - | Self::AzureBlob(_) - | Self::Gcs(_) => None, - } - } - - pub fn semantic_config(&self) -> Option<(f64, &str)> { - match self { - Self::ValkeySemantic { cache, .. } => Some(( - cache.backend().similarity_threshold(), - cache.backend().index_name(), - )), - Self::RedisSemantic { cache, .. } => Some(( - f64::from(cache.backend().similarity_threshold()), - cache.backend().index_name(), - )), - _ => None, - } - } - - pub fn index_name(&self) -> Option<&str> { - match self { - Self::RedisSemantic { cache, .. } => Some(cache.backend().index_name()), - _ => None, - } - } - - pub fn similarity_threshold(&self) -> Option { - match self { - Self::RedisSemantic { cache, .. } => { - Some(f64::from(cache.backend().similarity_threshold())) - } - Self::QdrantSemantic(cache) => Some(cache.backend().similarity_threshold()), - _ => None, - } - } - - pub fn collection_name(&self) -> Option<&str> { - match self { - Self::QdrantSemantic(cache) => Some(cache.backend().collection_name()), - _ => None, - } - } - - pub fn vector_size(&self) -> Option { - match self { - Self::QdrantSemantic(cache) => Some(cache.backend().vector_size()), - _ => None, - } - } - - pub fn embedding_model(&self) -> Option<&str> { - match self { - Self::QdrantSemantic(cache) => Some(cache.backend().embedder().model()), - _ => None, - } - } - - pub fn semantic_embedder(&self) -> Option<&PythonEmbedder> { - match self { - Self::RedisSemantic { embedder, .. } => Some(embedder), - _ => None, - } - } - pub fn embedder_object(&self) -> Option<&Py> { match self { Self::RedisSemantic { embedder, .. } => Some(embedder.object()), @@ -502,21 +323,71 @@ impl NativeResponseCache { } } + /// The prompt and metadata this backend would embed for `request`, if it has a prompt. + pub(super) fn embedding_input(&self, request: &NativeRequest) -> Option { + let context = match self { + Self::ValkeySemantic { scope, .. } => request.scoped_semantic(scope).context, + Self::RedisSemantic { .. } => request.semantic().context, + Self::Exact(_) | Self::QdrantSemantic(_) => return None, + }; + let prompt = litellm_cache::semantic::prompt_from_context(&context)?; + Some(EmbeddingInput { + prompt, + metadata: context.metadata, + }) + } + + /// Drives a semantic operation whose embedding comes from Python. + fn python_semantic<'py>( + &self, + py: Python<'py>, + operation: SemanticOperation, + ) -> PyResult> { + let (embedder, failure) = match self { + Self::ValkeySemantic { embedder, .. } => (embedder, EmbeddingFailure::Propagate), + Self::RedisSemantic { embedder, .. } => (embedder, EmbeddingFailure::Unavailable), + Self::Exact(_) | Self::QdrantSemantic(_) => { + return Err(pyo3::exceptions::PyRuntimeError::new_err( + "semantic execution requires a Python-embedded backend", + )); + } + }; + drive( + py, + SemanticExecution::new(self.clone(), embedder.clone(), failure, operation), + ) + } + pub fn lookup(&self, request: &NativeRequest, now: Duration) -> Result, Error> { match self { - Self::Memory(cache) => cache.lookup(&Self::exact(request), now), - Self::Redis { cache, .. } => cache.lookup(&Self::exact(request), now), - Self::S3(cache) => cache.lookup(&Self::exact(request), now), + Self::Exact(service) => service.cache.lookup(&request.exact(), now), Self::ValkeySemantic { cache, scope, .. } => { - cache.lookup(&Self::semantic(request, scope), now) + cache.lookup(&request.scoped_semantic(scope), now) + } + Self::RedisSemantic { cache, .. } => cache.lookup(&request.semantic(), now), + Self::QdrantSemantic(cache) => cache.lookup(&request.semantic(), now), + } + } + + /// `lookup` plus the similarity Python's semantic backend writes to the request metadata. + /// Exact backends report none. + pub fn lookup_semantic( + &self, + request: &NativeRequest, + now: Duration, + ) -> Result, Error> { + match self { + Self::Exact(service) => service + .cache + .lookup(&request.exact(), now) + .map(exact_lookup), + Self::ValkeySemantic { cache, scope, .. } => { + redis_family(cache.lookup_semantic(&request.scoped_semantic(scope), now)) } Self::RedisSemantic { cache, .. } => { - cache.lookup(&Self::semantic_request(request), now) + redis_family(cache.lookup_semantic(&request.semantic(), now)) } - Self::Gcs(cache) => cache.lookup(&Self::exact(request), now), - Self::QdrantSemantic(cache) => cache.lookup(&Self::semantic_request(request), now), - Self::Disk(cache) => cache.lookup(&Self::exact(request), now), - Self::AzureBlob(cache) => cache.lookup(&Self::exact(request), now), + Self::QdrantSemantic(cache) => cache.lookup_semantic(&request.semantic(), now), } } @@ -527,21 +398,12 @@ impl NativeResponseCache { now: Duration, ) -> Result<(), Error> { match self { - Self::Memory(cache) => cache.store(&Self::exact(request), response, now), - Self::Redis { cache, .. } => cache.store(&Self::exact(request), response, now), - Self::S3(cache) => cache.store(&Self::exact(request), response, now), + Self::Exact(service) => service.cache.store(&request.exact(), response, now), Self::ValkeySemantic { cache, scope, .. } => { - cache.store(&Self::semantic(request, scope), response, now) + cache.store(&request.scoped_semantic(scope), response, now) } - Self::RedisSemantic { cache, .. } => { - cache.store(&Self::semantic_request(request), response, now) - } - Self::Gcs(cache) => cache.store(&Self::exact(request), response, now), - Self::QdrantSemantic(cache) => { - cache.store(&Self::semantic_request(request), response, now) - } - Self::Disk(cache) => cache.store(&Self::exact(request), response, now), - Self::AzureBlob(cache) => cache.store(&Self::exact(request), response, now), + Self::RedisSemantic { cache, .. } => cache.store(&request.semantic(), response, now), + Self::QdrantSemantic(cache) => cache.store(&request.semantic(), response, now), } } @@ -551,29 +413,10 @@ impl NativeResponseCache { now: Duration, ) -> Result { match self { - Self::Memory(cache) => { - let requests = requests.iter().map(Self::exact).collect::>(); - cache.lookup_batch(&requests, now) - } - Self::Redis { cache, .. } => { - let requests = requests.iter().map(Self::exact).collect::>(); - cache.lookup_batch(&requests, now) - } - Self::S3(cache) => { - cache.lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) - } + Self::Exact(service) => service.cache.lookup_batch(&exact_requests(requests), now), Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => { Err(Error::UnsupportedOperation) } - Self::Gcs(cache) => { - cache.lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) - } - Self::Disk(cache) => { - cache.lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) - } - Self::AzureBlob(cache) => { - cache.lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) - } } } @@ -583,27 +426,64 @@ impl NativeResponseCache { now: Duration, ) -> Result, Error> { match self { - Self::Memory(cache) => cache.async_lookup(&Self::exact(request), now).await, - Self::Redis { cache, .. } => cache.async_lookup(&Self::exact(request), now).await, - Self::S3(cache) => cache.async_lookup(&Self::exact(request), now).await, + Self::Exact(service) => service.cache.async_lookup(&request.exact(), now).await, Self::ValkeySemantic { cache, scope, .. } => { cache - .async_lookup(&Self::semantic(request, scope), now) + .async_lookup(&request.scoped_semantic(scope), now) .await } - Self::RedisSemantic { cache, .. } => { + Self::RedisSemantic { cache, .. } => cache.async_lookup(&request.semantic(), now).await, + Self::QdrantSemantic(cache) => cache.async_lookup(&request.semantic(), now).await, + } + } + + pub async fn async_lookup_semantic( + &self, + request: &NativeRequest, + now: Duration, + ) -> Result, Error> { + match self { + Self::Exact(service) => service + .cache + .async_lookup(&request.exact(), now) + .await + .map(exact_lookup), + Self::ValkeySemantic { cache, scope, .. } => redis_family( cache - .async_lookup(&Self::semantic_request(request), now) - .await + .async_lookup_semantic(&request.scoped_semantic(scope), now) + .await, + ), + Self::RedisSemantic { cache, .. } => { + redis_family(cache.async_lookup_semantic(&request.semantic(), now).await) } Self::QdrantSemantic(cache) => { - cache - .async_lookup(&Self::semantic_request(request), now) - .await + cache.async_lookup_semantic(&request.semantic(), now).await + } + } + } + + pub(super) fn async_lookup_semantic_py<'py>( + &self, + py: Python<'py>, + request: NativeRequest, + ) -> PyResult> { + match self { + Self::Exact(_) | Self::QdrantSemantic(_) => { + let service = self.clone(); + litellm_host_python::run_async( + py, + async move { + service + .async_lookup_semantic(&request, now()) + .await + .map(SemanticReply::from) + }, + super::cache_error, + ) + } + Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } => { + self.python_semantic(py, SemanticOperation::LookupSemantic(request)) } - Self::Gcs(cache) => cache.async_lookup(&Self::exact(request), now).await, - Self::Disk(cache) => cache.async_lookup(&Self::exact(request), now).await, - Self::AzureBlob(cache) => cache.async_lookup(&Self::exact(request), now).await, } } @@ -613,42 +493,16 @@ impl NativeResponseCache { request: NativeRequest, ) -> PyResult> { match self { - Self::Memory(_) - | Self::Redis { .. } - | Self::S3(_) - | Self::Disk(_) - | Self::AzureBlob(_) - | Self::Gcs(_) => { + Self::Exact(_) | Self::QdrantSemantic(_) => { let service = self.clone(); litellm_host_python::run_async( py, - async move { service.async_lookup(&request, super::request::now()).await }, + async move { service.async_lookup(&request, now()).await }, super::cache_error, ) } - Self::ValkeySemantic { - cache, - embedder, - scope, - } => drive_semantic( - py, - SemanticEmbedExecution::lookup( - Arc::clone(cache.backend_arc()), - embedder.clone(), - Self::semantic(&request, scope), - ), - ), - Self::RedisSemantic { .. } => drive( - py, - SemanticBody::new(self.clone(), SemanticOperation::Lookup(request)), - ), - Self::QdrantSemantic(_) => { - let service = self.clone(); - litellm_host_python::run_async( - py, - async move { service.async_lookup(&request, super::request::now()).await }, - super::cache_error, - ) + Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } => { + self.python_semantic(py, SemanticOperation::Lookup(request)) } } } @@ -660,61 +514,29 @@ impl NativeResponseCache { now: Duration, ) -> Result<(), Error> { match self { - Self::Memory(cache) => { - cache - .async_store(&Self::exact(request), response, now) - .await - } - Self::Redis { - cache, - buffer: None, - } => { - cache - .async_store(&Self::exact(request), response, now) - .await - } - Self::Redis { - cache, - buffer: Some(buffer), - } => { - buffer - .async_store(cache, &Self::exact(request), response, now) - .await - } - Self::S3(cache) => { - cache - .async_store(&Self::exact(request), response, now) - .await - } + Self::Exact(service) => match &service.buffer { + None => { + service + .cache + .async_store(&request.exact(), response, now) + .await + } + Some(buffer) => { + buffer + .async_store(service.cache.as_ref(), &request.exact(), response, now) + .await + } + }, Self::ValkeySemantic { cache, scope, .. } => { cache - .async_store(&Self::semantic(request, scope), response, now) + .async_store(&request.scoped_semantic(scope), response, now) .await } Self::RedisSemantic { cache, .. } => { - cache - .async_store(&Self::semantic_request(request), response, now) - .await + cache.async_store(&request.semantic(), response, now).await } Self::QdrantSemantic(cache) => { - cache - .async_store(&Self::semantic_request(request), response, now) - .await - } - Self::Gcs(cache) => { - cache - .async_store(&Self::exact(request), response, now) - .await - } - Self::Disk(cache) => { - cache - .async_store(&Self::exact(request), response, now) - .await - } - Self::AzureBlob(cache) => { - cache - .async_store(&Self::exact(request), response, now) - .await + cache.async_store(&request.semantic(), response, now).await } } } @@ -726,51 +548,16 @@ impl NativeResponseCache { response: Value, ) -> PyResult> { match self { - Self::Memory(_) - | Self::Redis { .. } - | Self::S3(_) - | Self::Disk(_) - | Self::AzureBlob(_) - | Self::Gcs(_) => { + Self::Exact(_) | Self::QdrantSemantic(_) => { let service = self.clone(); litellm_host_python::run_async( py, - async move { - service - .async_store(&request, response, super::request::now()) - .await - }, + async move { service.async_store(&request, response, now()).await }, super::cache_error, ) } - Self::ValkeySemantic { - cache, - embedder, - scope, - } => drive_semantic( - py, - SemanticEmbedExecution::store( - Arc::clone(cache.backend_arc()), - embedder.clone(), - Self::semantic(&request, scope), - response, - ), - ), - Self::RedisSemantic { .. } => drive( - py, - SemanticBody::new(self.clone(), SemanticOperation::Store(request, response)), - ), - Self::QdrantSemantic(_) => { - let service = self.clone(); - litellm_host_python::run_async( - py, - async move { - service - .async_store(&request, response, super::request::now()) - .await - }, - super::cache_error, - ) + Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } => { + self.python_semantic(py, SemanticOperation::Store(request, response)) } } } @@ -781,37 +568,15 @@ impl NativeResponseCache { now: Duration, ) -> Result { match self { - Self::Memory(cache) => { - let requests = requests.iter().map(Self::exact).collect::>(); - cache.async_lookup_batch(&requests, now).await - } - Self::Redis { cache, .. } => { - let requests = requests.iter().map(Self::exact).collect::>(); - cache.async_lookup_batch(&requests, now).await - } - Self::S3(cache) => { - cache - .async_lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) + Self::Exact(service) => { + service + .cache + .async_lookup_batch(&exact_requests(requests), now) .await } Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => { Err(Error::UnsupportedOperation) } - Self::Gcs(cache) => { - cache - .async_lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) - .await - } - Self::Disk(cache) => { - cache - .async_lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) - .await - } - Self::AzureBlob(cache) => { - cache - .async_lookup_batch(&requests.iter().map(Self::exact).collect::>(), now) - .await - } } } @@ -821,55 +586,25 @@ impl NativeResponseCache { now: Duration, ) -> Result<(), Error> { match self { - Self::Memory(cache) => { + Self::Exact(service) => { let entries = entries .into_iter() - .map(|(request, value)| (Self::exact(&request), value)) + .map(|(request, value)| (request.exact(), value)) .collect(); - cache.async_store_batch(entries, now).await - } - Self::Redis { cache, .. } => { - let entries = entries - .into_iter() - .map(|(request, value)| (Self::exact(&request), value)) - .collect(); - cache.async_store_batch(entries, now).await - } - Self::S3(cache) => { - let entries = entries - .into_iter() - .map(|(request, value)| (Self::exact(&request), value)) - .collect(); - cache.async_store_batch(entries, now).await + service.cache.async_store_batch(entries, now).await } Self::ValkeySemantic { cache, scope, .. } => { let entries = entries .into_iter() - .map(|(request, value)| (Self::semantic(&request, scope), value)) + .map(|(request, value)| (request.scoped_semantic(scope), value)) .collect(); cache.async_store_batch(entries, now).await } - Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => { - Err(Error::UnsupportedOperation) - } - Self::Gcs(cache) => { + Self::RedisSemantic { .. } => Err(Error::UnsupportedOperation), + Self::QdrantSemantic(cache) => { let entries = entries .into_iter() - .map(|(request, value)| (Self::exact(&request), value)) - .collect(); - cache.async_store_batch(entries, now).await - } - Self::Disk(cache) => { - let entries = entries - .into_iter() - .map(|(request, value)| (Self::exact(&request), value)) - .collect(); - cache.async_store_batch(entries, now).await - } - Self::AzureBlob(cache) => { - let entries = entries - .into_iter() - .map(|(request, value)| (Self::exact(&request), value)) + .map(|(request, value)| (request.semantic(), value)) .collect(); cache.async_store_batch(entries, now).await } @@ -882,174 +617,80 @@ impl NativeResponseCache { entries: Vec<(NativeRequest, Value)>, ) -> PyResult> { match self { - Self::Memory(_) - | Self::Redis { .. } - | Self::S3(_) - | Self::Disk(_) - | Self::AzureBlob(_) - | Self::Gcs(_) => { + Self::Exact(_) | Self::QdrantSemantic(_) => { let service = self.clone(); litellm_host_python::run_async( py, - async move { - service - .async_store_batch(entries, super::request::now()) - .await - }, + async move { service.async_store_batch(entries, now()).await }, super::cache_error, ) } - Self::ValkeySemantic { - cache, - embedder, - scope, - } => { - let (requests, responses): (Vec<_>, Vec<_>) = entries - .into_iter() - .map(|(request, response)| (Self::semantic(&request, scope), response)) - .unzip(); - drive_semantic( - py, - SemanticEmbedExecution::store_batch( - Arc::clone(cache.backend_arc()), - embedder.clone(), - requests, - responses, - ), - ) + Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } => { + self.python_semantic(py, SemanticOperation::StoreBatch(entries.into())) } - Self::RedisSemantic { .. } => drive( - py, - SemanticBody::new(self.clone(), SemanticOperation::StoreBatch(entries.into())), - ), - Self::QdrantSemantic(_) => Err(super::cache_error(Error::UnsupportedOperation)), } } pub async fn async_flush(&self) -> Result<(), Error> { match self { - Self::Memory(cache) => cache.async_flush().await, - Self::Redis { cache, buffer } => { - if let Some(buffer) = buffer { + Self::Exact(service) => { + if let Some(buffer) = &service.buffer { buffer.clear()?; } - cache.async_flush().await + service.cache.async_flush().await } - Self::S3(cache) => cache.async_flush().await, Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => { Err(Error::UnsupportedOperation) } - Self::Gcs(cache) => cache.async_flush().await, - Self::Disk(cache) => cache.async_flush().await, - Self::AzureBlob(cache) => cache.async_flush().await, } } pub async fn test_connection(&self) -> Result { match self { - Self::Memory(cache) => cache.test_connection().await, - Self::Redis { cache, .. } => cache.test_connection().await, - Self::S3(cache) => cache.test_connection().await, - Self::ValkeySemantic { cache, .. } => cache.test_connection().await, - Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => { + Self::Exact(service) => match &service.probe { + Some(probe) => probe.test_connection().await, + None => Err(Error::UnsupportedOperation), + }, + Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => { Err(Error::UnsupportedOperation) } - Self::Gcs(cache) => cache.test_connection().await, - Self::Disk(cache) => cache.test_connection().await, - Self::AzureBlob(cache) => cache.test_connection().await, } } pub(super) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { match self { - Self::ValkeySemantic { embedder, .. } => embedder.traverse(visit)?, - Self::RedisSemantic { embedder, .. } => embedder.traverse(visit)?, - _ => {} - } - Ok(()) - } - - pub fn gcs_backend(&self) -> Option<&GcsCache> { - match self { - Self::Gcs(cache) => Some(cache.backend()), - _ => None, + Self::ValkeySemantic { embedder, .. } | Self::RedisSemantic { embedder, .. } => { + embedder.traverse(visit) + } + Self::Exact(_) | Self::QdrantSemantic(_) => Ok(()), } } } -#[cfg(test)] -mod tests { - use litellm_cache_response::{CacheControls, CacheKeyInput, cache_key}; - use serde_json::json; - use sha2::{Digest, Sha256}; +fn exact_requests(requests: &[NativeRequest]) -> Vec { + requests.iter().map(NativeRequest::exact).collect() +} - use super::*; +/// What `lookup_semantic` hands Python: the response and the similarity to stamp, if any. +#[derive(serde::Serialize)] +pub(super) struct SemanticReply(pub(super) Option, pub(super) Option); - fn native_request(key: CacheKeyInput, metadata: Value) -> NativeRequest { - NativeRequest { - key, - controls: CacheControls::default(), - ttl: None, - max_age: None, - messages: Some(json!([{"role": "user", "content": "prompt"}])), - input: None, - metadata: Some(metadata), - litellm_metadata: None, - litellm_params: None, - scope: None, - } - } - - #[test] - fn semantic_key_matches_python_scope_material() { - let key = CacheKeyInput { - fields: vec![ - CacheKeyField { - name: "model".to_owned(), - value: Some("gpt-4.1".to_owned()), - api_parameter: true, - internal_parameter: false, - }, - CacheKeyField { - name: "messages".to_owned(), - value: Some("prompt".to_owned()), - api_parameter: true, - internal_parameter: false, - }, - ], - ..Default::default() - }; - let request = native_request( - key, - json!({"user_api_key": "k1", "user_api_key_team_id": null}), - ); - let expected = format!("{:x}", Sha256::digest(b"model: gpt-4.1user_api_key: k1")); - assert_eq!(cache_key(&semantic_key(&request, "key")), expected); - - let end_user_request = native_request( - request.key.clone(), - json!({"user_api_key": "k1", "user_api_key_end_user_id": "u1"}), - ); - let expected = format!( - "{:x}", - Sha256::digest(b"model: gpt-4.1user_api_key: k1user_api_key_end_user_id: u1") - ); - assert_eq!( - cache_key(&semantic_key(&end_user_request, "end_user")), - expected - ); - - let preset_request = native_request( - CacheKeyInput { - preset: Some("preset-key".to_owned()), - ..Default::default() - }, - json!({"user_api_key": "k1"}), - ); - assert_eq!( - semantic_key(&preset_request, "end_user").preset.as_deref(), - Some("preset-key") - ); - assert!(semantic_key(&preset_request, "end_user").fields.is_empty()); +impl From> for SemanticReply { + fn from(lookup: SemanticLookup) -> Self { + Self(lookup.value, lookup.similarity) } } + +fn exact_lookup(value: Option) -> SemanticLookup { + SemanticLookup { + value, + similarity: None, + } +} + +/// Python's Redis and Valkey semantic caches catch every lookup failure and stamp `0.0`. +fn redis_family( + lookup: Result, Error>, +) -> Result, Error> { + Ok(lookup.unwrap_or_else(|_| SemanticLookup::miss(Some(0.0)))) +} diff --git a/litellm-rust/crates/python-bridge/src/cache/request.rs b/litellm-rust/crates/python-bridge/src/cache/request.rs index 3b4b910c1f0..627bf9f1840 100644 --- a/litellm-rust/crates/python-bridge/src/cache/request.rs +++ b/litellm-rust/crates/python-bridge/src/cache/request.rs @@ -1,7 +1,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use litellm_cache::ExactCacheContext; -use litellm_cache_response::{CacheControls, CacheKeyInput, ResponseCacheRequest}; +use litellm_cache::{ExactCacheContext, SemanticCacheContext}; +use litellm_cache_response::{CacheControls, CacheKeyField, CacheKeyInput, ResponseCacheRequest}; use litellm_host_python::from_py; use pyo3::{exceptions::PyValueError, prelude::*}; use serde::Deserialize; @@ -36,6 +36,99 @@ pub(super) struct NativeRequest { pub(super) scope: Option, } +impl NativeRequest { + pub(super) fn exact(&self) -> ResponseCacheRequest { + ResponseCacheRequest { + key: self.key.clone(), + controls: self.controls, + context: ExactCacheContext { ttl: self.ttl }, + max_age: self.max_age, + } + } + + /// The request as a semantic backend that keys on the caller's scope sees it. + pub(super) fn semantic(&self) -> ResponseCacheRequest { + self.semantic_with(self.key.clone(), self.scope.clone()) + } + + /// The request keyed the way Python's Valkey semantic cache keys it: prompt fields drop out + /// and the tenant identifiers for `scope` join the key. + pub(super) fn scoped_semantic( + &self, + scope: &str, + ) -> ResponseCacheRequest { + self.semantic_with(semantic_key(self, scope), Some(scope.to_owned())) + } + + fn semantic_with( + &self, + key: CacheKeyInput, + scope: Option, + ) -> ResponseCacheRequest { + ResponseCacheRequest { + key, + controls: self.controls, + context: SemanticCacheContext { + input: self.input.clone(), + messages: self.messages.clone(), + metadata: self.metadata.clone(), + scope, + ttl: self.ttl, + }, + max_age: self.max_age, + } + } +} + +fn semantic_key(request: &NativeRequest, scope: &str) -> CacheKeyInput { + let mut key = request.key.clone(); + if key.preset.is_some() { + return key; + } + key.fields + .retain(|field| !matches!(field.name.as_str(), "messages" | "prompt" | "input")); + const TENANT: [&str; 3] = [ + "user_api_key", + "user_api_key_team_id", + "user_api_key_org_id", + ]; + let end_user = (scope == "end_user").then_some("user_api_key_end_user_id"); + for name in TENANT.into_iter().chain(end_user) { + let sources = [ + request.metadata.as_ref(), + request.litellm_metadata.as_ref(), + request + .litellm_params + .as_ref() + .and_then(|params| params.get("metadata")), + request + .litellm_params + .as_ref() + .and_then(|params| params.get("litellm_metadata")), + ]; + let Some(value) = sources.into_iter().flatten().find_map(|source| { + source + .as_object() + .and_then(|values| values.get(name)) + .filter(|value| !value.is_null()) + }) else { + continue; + }; + let value = match value { + Value::Null => continue, + Value::String(text) => text.clone(), + other => other.to_string(), + }; + key.fields.push(CacheKeyField { + name: name.to_owned(), + value: Some(value), + api_parameter: true, + internal_parameter: false, + }); + } + key +} + pub(super) fn request(value: &Bound<'_, PyAny>) -> PyResult { let input: RequestInput = from_py(value)?; request_input(input) @@ -76,3 +169,90 @@ pub(super) fn now() -> Duration { .duration_since(UNIX_EPOCH) .unwrap_or_default() } + +#[cfg(test)] +mod tests { + use litellm_cache_response::{CacheControls, CacheKeyInput, cache_key}; + use serde_json::json; + use sha2::{Digest, Sha256}; + + use super::*; + + fn native_request(key: CacheKeyInput, metadata: Value) -> NativeRequest { + NativeRequest { + key, + controls: CacheControls::default(), + ttl: None, + max_age: None, + messages: Some(json!([{"role": "user", "content": "prompt"}])), + input: None, + metadata: Some(metadata), + litellm_metadata: None, + litellm_params: None, + scope: None, + } + } + + #[test] + fn semantic_key_matches_python_scope_material() { + let key = CacheKeyInput { + fields: vec![ + CacheKeyField { + name: "model".to_owned(), + value: Some("gpt-4.1".to_owned()), + api_parameter: true, + internal_parameter: false, + }, + CacheKeyField { + name: "messages".to_owned(), + value: Some("prompt".to_owned()), + api_parameter: true, + internal_parameter: false, + }, + ], + ..Default::default() + }; + let request = native_request( + key, + json!({"user_api_key": "k1", "user_api_key_team_id": null}), + ); + let expected = format!("{:x}", Sha256::digest(b"model: gpt-4.1user_api_key: k1")); + assert_eq!(cache_key(&semantic_key(&request, "key")), expected); + assert_eq!(cache_key(&request.scoped_semantic("key").key), expected); + + let end_user_request = native_request( + request.key.clone(), + json!({"user_api_key": "k1", "user_api_key_end_user_id": "u1"}), + ); + let expected = format!( + "{:x}", + Sha256::digest(b"model: gpt-4.1user_api_key: k1user_api_key_end_user_id: u1") + ); + assert_eq!( + cache_key(&semantic_key(&end_user_request, "end_user")), + expected + ); + + let preset_request = native_request( + CacheKeyInput { + preset: Some("preset-key".to_owned()), + ..Default::default() + }, + json!({"user_api_key": "k1"}), + ); + assert_eq!( + semantic_key(&preset_request, "end_user").preset.as_deref(), + Some("preset-key") + ); + assert!(semantic_key(&preset_request, "end_user").fields.is_empty()); + assert_eq!(preset_request.semantic().context.scope, None); + assert_eq!( + preset_request + .scoped_semantic("end_user") + .context + .scope + .as_deref(), + Some("end_user") + ); + } +} diff --git a/litellm-rust/crates/python-bridge/src/cache/semantic.rs b/litellm-rust/crates/python-bridge/src/cache/semantic.rs index 934de01e721..bea58b88885 100644 --- a/litellm-rust/crates/python-bridge/src/cache/semantic.rs +++ b/litellm-rust/crates/python-bridge/src/cache/semantic.rs @@ -1,7 +1,6 @@ -use std::collections::VecDeque; +use std::{collections::VecDeque, time::Duration}; use litellm_cache::Error; -use litellm_cache_redis_semantic::prompt_from_context; use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async}; use pyo3::{ PyTraverseError, PyVisit, @@ -13,36 +12,123 @@ use serde_json::Value; use super::{ cache_error, embedder::{PythonEmbedder, with_prepared_embedding}, - native::NativeResponseCache, + native::{NativeResponseCache, SemanticReply}, request::{NativeRequest, now}, }; pub(super) enum SemanticOperation { Lookup(NativeRequest), + /// A lookup that also reports the similarity, as `SemanticReply`. + LookupSemantic(NativeRequest), Store(NativeRequest, Value), StoreBatch(VecDeque<(NativeRequest, Value)>), } +/// What an exception from the Python embedder means for the operation. +#[derive(Clone, Copy)] +pub(super) enum EmbeddingFailure { + /// Raise the Python exception unchanged. + Propagate, + /// Treat the embedding as unavailable and let the backend report that. + Unavailable, +} + enum Phase { Start, AwaitingEmbedding, AwaitingBackend, } -pub(super) struct SemanticBody { +/// Runs a semantic cache operation whose embedding comes from Python: await the Python +/// embedder in the caller's event loop, seed the native backend with the vector, await the +/// backend, and repeat for each entry of a batch. +pub(super) struct SemanticExecution { service: NativeResponseCache, + embedder: PythonEmbedder, + failure: EmbeddingFailure, operation: SemanticOperation, pending: Option<(NativeRequest, Option)>, phase: Phase, + now: Duration, } -impl SemanticBody { - pub(super) fn new(service: NativeResponseCache, operation: SemanticOperation) -> Self { +impl SemanticExecution { + pub(super) fn new( + service: NativeResponseCache, + embedder: PythonEmbedder, + failure: EmbeddingFailure, + operation: SemanticOperation, + ) -> Self { Self { service, + embedder, + failure, operation, pending: None, phase: Phase::Start, + now: now(), + } + } + + /// Takes the next entry of the operation; `None` once a batch is exhausted. + fn next_pending(&mut self) -> Option<(NativeRequest, Option)> { + match &mut self.operation { + SemanticOperation::Lookup(request) | SemanticOperation::LookupSemantic(request) => { + Some((request.clone(), None)) + } + SemanticOperation::Store(request, response) => { + Some((request.clone(), Some(std::mem::take(response)))) + } + SemanticOperation::StoreBatch(queue) => queue + .pop_front() + .map(|(request, response)| (request, Some(response))), + } + } + + fn start(&mut self, py: Python<'_>) -> PyResult { + let Some(pending) = self.next_pending() else { + return Ok(ExecutionStep::Return(py.None())); + }; + let (request, response) = &pending; + let enabled = match response { + None => request.controls.reads(), + Some(_) => request.controls.writes(), + }; + let input = enabled + .then(|| self.service.embedding_input(request)) + .flatten(); + self.pending = Some(pending); + let Some(input) = input else { + return self.backend_step(py, Err(Error::Unavailable)); + }; + let awaitable = + self.embedder + .async_embedding(py, &input.prompt, input.metadata.as_ref())?; + self.phase = Phase::AwaitingEmbedding; + Ok(ExecutionStep::Await(awaitable)) + } + + fn embedded(&mut self, py: Python<'_>, result: PyResult>) -> PyResult { + let seed = match result { + Ok(vector) => { + PythonEmbedder::extract(vector.into_bound(py)).map_err(|_| Error::Unavailable) + } + Err(error) => match self.embedding_failure() { + EmbeddingFailure::Propagate => return Err(error), + EmbeddingFailure::Unavailable if error.is_instance_of::(py) => { + Err(Error::Unavailable) + } + EmbeddingFailure::Unavailable => return Err(error), + }, + }; + self.backend_step(py, seed) + } + + /// Python's semantic lookups catch embedding errors and stamp a similarity of `0.0`. + fn embedding_failure(&self) -> EmbeddingFailure { + match self.operation { + SemanticOperation::LookupSemantic(_) => EmbeddingFailure::Unavailable, + _ => self.failure, } } @@ -56,118 +142,70 @@ impl SemanticBody { PyRuntimeError::new_err("semantic execution resumed without a pending operation") })?; let service = self.service.clone(); + let now = self.now; + let with_similarity = matches!(self.operation, SemanticOperation::LookupSemantic(_)); let future = async move { match response { - None => service.async_lookup(&request, now()).await, - Some(response) => service - .async_store(&request, response, now()) + None if with_similarity => service + .async_lookup_semantic(&request, now) .await - .map(|_| None), + .map(|lookup| Reply::Semantic(lookup.into())), + None => service.async_lookup(&request, now).await.map(Reply::Plain), + Some(response) => service + .async_store(&request, response, now) + .await + .map(|_| Reply::Plain(None)), } }; let awaitable = run_async(py, with_prepared_embedding(seed, future), cache_error)?; Ok(ExecutionStep::Await(awaitable.unbind())) } + + fn resume_py( + &mut self, + py: Python<'_>, + result: Option>>, + ) -> PyResult { + match (&self.phase, result) { + (Phase::Start, None) => self.start(py), + (Phase::AwaitingEmbedding, Some(result)) => self.embedded(py, result), + (Phase::AwaitingBackend, Some(Err(error))) => Err(error), + (Phase::AwaitingBackend, Some(Ok(value))) => { + let more = matches!( + &self.operation, + SemanticOperation::StoreBatch(queue) if !queue.is_empty() + ); + if more { + self.phase = Phase::Start; + return self.start(py); + } + Ok(ExecutionStep::Return(value)) + } + _ => Err(PyRuntimeError::new_err( + "invalid semantic cache execution state", + )), + } + } } -impl ExecutionBody for SemanticBody { - fn resume(&mut self, mut result: Option>>) -> PyResult { - Python::attach(|py| { - loop { - match self.phase { - Phase::Start => { - if result.is_some() { - return Err(PyRuntimeError::new_err( - "semantic execution received a result before starting", - )); - } - if self.pending.is_none() { - match &mut self.operation { - SemanticOperation::Lookup(request) => { - self.pending = Some((request.clone(), None)); - } - SemanticOperation::Store(request, response) => { - let response = std::mem::replace(response, Value::Null); - self.pending = Some((request.clone(), Some(response))); - } - SemanticOperation::StoreBatch(queue) => { - let Some((request, response)) = queue.pop_front() else { - return Ok(ExecutionStep::Return(py.None())); - }; - self.pending = Some((request, Some(response))); - } - } - } - let (request, _) = self.pending.as_ref().ok_or_else(|| { - PyRuntimeError::new_err("semantic execution has no pending operation") - })?; - let semantic = NativeResponseCache::semantic_request(request); - let Some(prompt) = prompt_from_context(&semantic.context) else { - return self.backend_step(py, Err(Error::Unavailable)); - }; - let embedder = self.service.semantic_embedder().ok_or_else(|| { - PyRuntimeError::new_err( - "semantic execution requires a redis-semantic backend", - ) - })?; - let coroutine = embedder.async_embedding_coroutine( - py, - &prompt, - semantic.context.metadata.as_ref(), - )?; - self.phase = Phase::AwaitingEmbedding; - return Ok(ExecutionStep::Await(coroutine)); - } - Phase::AwaitingEmbedding => { - let result = result.take().ok_or_else(|| { - PyRuntimeError::new_err( - "semantic execution expected an embedding result", - ) - })?; - let seed = match result { - Ok(value) => PythonEmbedder::extract(value.into_bound(py)) - .map_err(|_| Error::Unavailable), - Err(error) => { - if !error.is_instance_of::(py) { - return Err(error); - } - Err(Error::Unavailable) - } - }; - return self.backend_step(py, seed); - } - Phase::AwaitingBackend => { - let result = result.take().ok_or_else(|| { - PyRuntimeError::new_err("semantic execution expected a backend result") - })?; - let value = match result { - Ok(value) => value, - Err(error) => return Err(error), - }; - let more = matches!( - &self.operation, - SemanticOperation::StoreBatch(queue) if !queue.is_empty() - ); - if more { - self.phase = Phase::Start; - continue; - } - return Ok(ExecutionStep::Return(value)); - } - } - } - }) +#[derive(serde::Serialize)] +#[serde(untagged)] +enum Reply { + Plain(Option), + Semantic(SemanticReply), +} + +impl ExecutionBody for SemanticExecution { + fn resume(&mut self, result: Option>>) -> PyResult { + Python::attach(|py| self.resume_py(py, result)) } fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { - if let Some(embedder) = self.service.semantic_embedder() { - embedder.traverse(visit)?; - } - Ok(()) + self.embedder.traverse(visit) } } -pub(super) fn drive(py: Python<'_>, body: SemanticBody) -> PyResult> { +pub(super) fn drive(py: Python<'_>, body: SemanticExecution) -> PyResult> { let execution = Py::new(py, Execution::new(body))?; py.import("litellm.rust_bridge.lifecycle")? .getattr("drive")? diff --git a/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs b/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs deleted file mode 100644 index 24caf3374d6..00000000000 --- a/litellm-rust/crates/python-bridge/src/cache/semantic_step.rs +++ /dev/null @@ -1,249 +0,0 @@ -use std::{sync::Arc, time::Duration}; - -use litellm_cache::SemanticCacheContext; -use litellm_cache_response::{ResponseCache, ResponseCacheCodec, ResponseCacheRequest}; -use litellm_cache_valkey_semantic::{PreparedEmbedding, ValkeySemanticCache, prompt_from_context}; -use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async}; -use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*}; -use serde_json::Value; - -use super::{cache_error, embedder::PythonEmbedder}; - -pub(super) enum Op { - Lookup, - Store(Value), - StoreBatch(Vec), -} - -#[derive(Clone, Copy)] -enum State { - Start, - AwaitingEmbedding, - AwaitingStorage, - Done, -} - -pub(super) struct SemanticEmbedExecution { - backend: Arc>, - embedder: PythonEmbedder, - requests: Vec>, - op: Op, - now: Option, - prepared: Vec>>, - index: usize, - state: State, -} - -impl SemanticEmbedExecution { - pub(super) fn lookup( - backend: Arc>, - embedder: PythonEmbedder, - request: ResponseCacheRequest, - ) -> Self { - Self { - backend, - embedder, - requests: vec![request], - op: Op::Lookup, - now: None, - prepared: vec![None], - index: 0, - state: State::Start, - } - } - - pub(super) fn store( - backend: Arc>, - embedder: PythonEmbedder, - request: ResponseCacheRequest, - response: Value, - ) -> Self { - Self { - backend, - embedder, - requests: vec![request], - op: Op::Store(response), - now: None, - prepared: vec![None], - index: 0, - state: State::Start, - } - } - - pub(super) fn store_batch( - backend: Arc>, - embedder: PythonEmbedder, - requests: Vec>, - responses: Vec, - ) -> Self { - Self { - backend, - embedder, - prepared: vec![None; requests.len()], - requests, - op: Op::StoreBatch(responses), - now: None, - index: 0, - state: State::Start, - } - } - - fn start(&mut self, py: Python<'_>) -> PyResult { - if self.now.is_none() { - self.now = Some(super::request::now()); - } - while self.index < self.requests.len() { - let request = &self.requests[self.index]; - let enabled = match &self.op { - Op::Lookup => request.controls.reads(), - Op::Store(_) | Op::StoreBatch(_) => request.controls.writes(), - }; - if !enabled { - self.index += 1; - continue; - } - let Some(prompt) = prompt_from_context(&request.context) else { - self.index += 1; - continue; - }; - let metadata = request.context.metadata.clone(); - let awaitable = self - .embedder - .async_embed_awaitable(py, &prompt, &metadata)?; - self.state = State::AwaitingEmbedding; - return Ok(ExecutionStep::Await(awaitable.unbind())); - } - self.state = State::AwaitingStorage; - self.storage_step(py) - } - - fn storage_step(&self, py: Python<'_>) -> PyResult { - let requests = self.requests.clone(); - let prepared = self.prepared.clone(); - let backend = Arc::clone(&self.backend); - let now = self - .now - .ok_or_else(|| PyRuntimeError::new_err("semantic cache timestamp is unavailable"))?; - let awaitable = match &self.op { - Op::Lookup => { - let Some(request) = requests.into_iter().next() else { - return Err(PyRuntimeError::new_err( - "semantic lookup requires one request", - )); - }; - match prepared.into_iter().next().flatten() { - Some(values) => { - let backend = backend.with_embedder(PreparedEmbedding(values)); - let cache = Arc::new(ResponseCache::new(Arc::new(backend))); - run_async( - py, - async move { cache.async_lookup(&request, now).await }, - cache_error, - )? - } - None => { - let cache = Arc::new(ResponseCache::new(backend)); - run_async( - py, - async move { cache.async_lookup(&request, now).await }, - cache_error, - )? - } - } - } - Op::Store(response) => { - let Some(request) = requests.into_iter().next() else { - return Err(PyRuntimeError::new_err( - "semantic store requires one request", - )); - }; - let response = response.clone(); - match prepared.into_iter().next().flatten() { - Some(values) => { - let backend = backend.with_embedder(PreparedEmbedding(values)); - let cache = Arc::new(ResponseCache::new(Arc::new(backend))); - run_async( - py, - async move { cache.async_store(&request, response, now).await }, - cache_error, - )? - } - None => { - let cache = Arc::new(ResponseCache::new(backend)); - run_async( - py, - async move { cache.async_store(&request, response, now).await }, - cache_error, - )? - } - } - } - Op::StoreBatch(responses) => { - let responses = responses.clone(); - run_async( - py, - async move { - for ((request, response), prepared) in - requests.into_iter().zip(responses).zip(prepared) - { - let Some(values) = prepared else { - continue; - }; - let backend = backend.with_embedder(PreparedEmbedding(values)); - let cache = ResponseCache::new(Arc::new(backend)); - cache.async_store(&request, response, now).await?; - } - Ok(()) - }, - cache_error, - )? - } - }; - Ok(ExecutionStep::Await(awaitable.unbind())) - } - - fn resume_py( - &mut self, - py: Python<'_>, - result: Option>>, - ) -> PyResult { - match (self.state, result) { - (State::Start, None) => self.start(py), - (State::AwaitingEmbedding, Some(Ok(value))) => { - let values = value.bind(py).extract::>()?; - self.prepared[self.index] = - Some(values.into_iter().map(|value| value as f32).collect()); - self.index += 1; - self.start(py) - } - (State::AwaitingStorage, Some(Ok(value))) => { - self.state = State::Done; - Ok(ExecutionStep::Return(value)) - } - (_, Some(Err(error))) => Err(error), - _ => Err(PyRuntimeError::new_err( - "invalid semantic cache execution state", - )), - } - } -} - -impl ExecutionBody for SemanticEmbedExecution { - fn resume(&mut self, result: Option>>) -> PyResult { - Python::attach(|py| self.resume_py(py, result)) - } - - fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { - self.embedder.traverse(visit) - } -} - -pub(super) fn drive_semantic<'py>( - py: Python<'py>, - body: SemanticEmbedExecution, -) -> PyResult> { - let execution = Py::new(py, Execution::new(body))?; - py.import("litellm.rust_bridge.lifecycle")? - .getattr("drive")? - .call1((execution,)) -} diff --git a/litellm-rust/crates/python-bridge/src/coercion.rs b/litellm-rust/crates/python-bridge/src/coercion.rs index bb5b8b2d454..1b0b073b3d1 100644 --- a/litellm-rust/crates/python-bridge/src/coercion.rs +++ b/litellm-rust/crates/python-bridge/src/coercion.rs @@ -1,7 +1,4 @@ -use std::collections::BTreeSet; - use litellm_core_utils::serde_compat::parse_str_bool; -use litellm_http::SslVerify; use pyo3::{ exceptions::{PyAttributeError, PyRuntimeError, PyValueError}, prelude::*, @@ -33,36 +30,52 @@ impl From for PyErr { } } -pub(crate) struct Truthy(pub bool); -pub(crate) struct ExactTrue(pub bool); -pub(crate) struct StrBool(pub Option); -pub(crate) struct OptionalStrictString(pub Option); -pub(crate) struct FalsyOptionalString(pub Option); -pub(crate) struct TuningString(pub Option); -pub(crate) struct StringCollection(pub Vec); -pub(crate) struct SslVerifyInput(pub Option); +pub(crate) struct FieldSpec { + name: &'static str, + decode: fn(&Field<'_>) -> Result, +} + +impl FieldSpec { + pub(crate) const fn new( + name: &'static str, + decode: fn(&Field<'_>) -> Result, + ) -> Self { + Self { name, decode } + } + + pub(crate) fn read( + &self, + snapshot: &Bound<'_, PyAny>, + group: &'static str, + ) -> Result { + (self.decode)(&Field::read(snapshot, group, self.name)?) + } +} pub(crate) struct Field<'py> { - path: &'static str, + group: &'static str, + name: &'static str, value: Bound<'py, PyAny>, } impl<'py> Field<'py> { - pub(crate) fn new(path: &'static str, value: Bound<'py, PyAny>) -> Self { - Self { path, value } + pub(crate) fn new(group: &'static str, name: &'static str, value: Bound<'py, PyAny>) -> Self { + Self { group, name, value } } + /// Reads `snapshot.`, distinguishing a field the accessor never declared from a + /// descriptor that raised `AttributeError`. pub(crate) fn read( snapshot: &Bound<'py, PyAny>, - path: &'static str, + group: &'static str, + name: &'static str, ) -> Result { - let name = path.rsplit('.').next().unwrap_or(path); match snapshot.getattr(name) { - Ok(value) => Ok(Self::new(path, value)), + Ok(value) => Ok(Self::new(group, name, value)), Err(error) if error.is_instance_of::(snapshot.py()) => { match Self::missing_field(snapshot, name) { Ok(true) => Err(ProjectionError::InternalSchemaFailure(format!( - "{path}: missing snapshot field" + "{group}.{name}: missing snapshot field" ))), _ => Err(error.into()), } @@ -84,27 +97,49 @@ impl<'py> Field<'py> { && getter.is(object.getattr("__getattribute__")?)) } - fn expected(&self, expected: &'static str) -> Result { + pub(crate) fn path(&self) -> String { + format!("{}.{}", self.group, self.name) + } + + /// A member of this field's collection, reported under the same path. + pub(crate) fn member(&self, value: Bound<'py, PyAny>) -> Self { + Self::new(self.group, self.name, value) + } + + pub(crate) fn expected(&self, expected: &str) -> Result { Ok(format!( "{}: expected {expected}, got {}", - self.path, + self.path(), self.value.get_type().name()? )) } - fn invalid(&self, expected: &'static str) -> ProjectionError { + pub(crate) fn invalid(&self, expected: &str) -> ProjectionError { match self.expected(expected) { Ok(message) => ProjectionError::InvalidConfiguration(message), Err(error) => error, } } - pub(crate) fn truthy(&self) -> Result { - Ok(Truthy(self.value.is_truthy()?)) + pub(crate) fn value(&self) -> &Bound<'py, PyAny> { + &self.value } - pub(crate) fn exact_true(&self) -> ExactTrue { - ExactTrue(self.value.is(PyBool::new(self.value.py(), true))) + pub(crate) fn truthy(&self) -> Result { + Ok(self.value.is_truthy()?) + } + + pub(crate) fn exact_true(&self) -> bool { + self.value.is(PyBool::new(self.value.py(), true)) + } + + pub(crate) fn schema_bool(&self) -> Result { + if !self.value.is_instance_of::() { + return Err(ProjectionError::InternalSchemaFailure( + self.expected("a Boolean")?, + )); + } + Ok(self.exact_true()) } pub(crate) fn strict_string(&self) -> Result { @@ -124,108 +159,366 @@ impl<'py> Field<'py> { self.strict_string() } - pub(crate) fn schema_bool(&self) -> Result { - if !self.value.is_instance_of::() { - return Err(ProjectionError::InternalSchemaFailure( - self.expected("a Boolean")?, - )); - } - Ok(self.exact_true().0) - } - - pub(crate) fn str_bool(&self) -> Result { + pub(crate) fn str_bool(&self) -> Result, ProjectionError> { if self.value.is_none() { - return Ok(StrBool(None)); + return Ok(None); } - Ok(StrBool(parse_str_bool(&self.strict_string()?))) + Ok(parse_str_bool(&self.strict_string()?)) } - pub(crate) fn optional_strict_string(&self) -> Result { + pub(crate) fn optional_strict_string(&self) -> Result, ProjectionError> { if self.value.is_none() { - return Ok(OptionalStrictString(None)); + return Ok(None); } - self.strict_string().map(Some).map(OptionalStrictString) + self.strict_string().map(Some) } - pub(crate) fn falsy_optional_string(&self) -> Result { - if !self.truthy()?.0 { - return Ok(FalsyOptionalString(None)); + pub(crate) fn falsy_optional_string(&self) -> Result, ProjectionError> { + if !self.truthy()? { + return Ok(None); } - self.strict_string().map(Some).map(FalsyOptionalString) + self.strict_string().map(Some) } - pub(crate) fn tuning_string(&self) -> Result { - if !self.truthy()?.0 || !self.value.is_instance_of::() { - return Ok(TuningString(None)); + pub(crate) fn tuning_string(&self) -> Result, ProjectionError> { + if !self.truthy()? || !self.value.is_instance_of::() { + return Ok(None); } - self.strict_string().map(Some).map(TuningString) + self.strict_string().map(Some) } - pub(crate) fn string_collection(&self) -> Result { - if !self.truthy()?.0 { - return Ok(StringCollection(Vec::new())); + pub(crate) fn string_collection(&self) -> Result, ProjectionError> { + if !self.truthy()? { + return Ok(Vec::new()); } if self.value.is_instance_of::() { - return self - .strict_string() - .map(|value| StringCollection(vec![value])); + return self.strict_string().map(|value| vec![value]); } - let values = self - .value + self.value .try_iter()? .filter_map(|item| { let member = match item { - Ok(value) => Self::new(self.path, value), + Ok(value) => self.member(value), Err(error) => return Some(Err(error.into())), }; match member.truthy() { - Ok(Truthy(false)) => None, - Ok(Truthy(true)) => Some(member.strict_string()), + Ok(false) => None, + Ok(true) => Some(member.strict_string()), Err(error) => Some(Err(error)), } }) - .collect::, ProjectionError>>()?; - Ok(StringCollection(values)) + .collect() } - pub(crate) fn host_collection(&self) -> Result { - let values = self - .string_collection()? - .0 - .into_iter() - .map(|host| litellm_http::media::normalize_host(&host)) - .collect::>(); - Ok(StringCollection(values.into_iter().collect())) - } - - pub(crate) fn ssl_verify(&self) -> Result { + pub(crate) fn optional_string_collection( + &self, + ) -> Result>, ProjectionError> { if self.value.is_none() { - return Ok(SslVerifyInput(None)); + return Ok(None); } - if self.value.is_instance_of::() { - return Ok(SslVerifyInput(Some(if self.exact_true().0 { - SslVerify::Enabled - } else { - SslVerify::Disabled - }))); - } - if self.value.is_instance_of::() { - let parsed = match self.str_bool()?.0 { - Some(true) => SslVerify::Enabled, - Some(false) => SslVerify::Disabled, - None => SslVerify::CaBundle(self.strict_string()?.into()), - }; - return Ok(SslVerifyInput(Some(parsed))); - } - let context = self.value.py().import("ssl")?.getattr("SSLContext")?; - if self.value.is_instance(&context)? { - return Err(ProjectionError::UnsupportedLiveObject(self.expected( - "a Boolean, Boolean string, CA path, or None; live SSLContext is unsupported", - )?)); - } - Err(self.invalid("a Boolean, Boolean string, CA path, or None")) + self.string_collection().map(Some) + } + + pub(crate) fn python_binding(&self) -> Option> { + (!self.value.is_none()).then(|| self.value.clone().unbind()) } } #[cfg(test)] -mod tests; +mod tests { + use std::ffi::CString; + + use pyo3::{ + exceptions::{PyLookupError, PyRuntimeError, PyValueError}, + types::PyDict, + }; + use rstest::rstest; + + use super::*; + + fn evaluate<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyAny> { + py.eval(&CString::new(source).unwrap(), None, None).unwrap() + } + + #[rstest] + #[case("None", false, false)] + #[case("False", false, false)] + #[case("True", true, true)] + #[case("0", false, false)] + #[case("1", true, false)] + #[case("''", false, false)] + #[case("'false'", true, false)] + #[case("[]", false, false)] + #[case("[0]", true, false)] + #[case("{}", false, false)] + #[case("object()", true, false)] + fn boolean_operations_have_distinct_python_semantics( + #[case] source: &str, + #[case] truth: bool, + #[case] exact: bool, + ) { + Python::initialize(); + Python::attach(|py| { + let value = evaluate(py, source); + let field = Field::new("test", "flag", value.clone()); + assert_eq!(field.truthy().unwrap(), truth); + assert_eq!(field.exact_true(), exact); + assert_eq!( + field.truthy().unwrap(), + py.import("builtins") + .unwrap() + .getattr("bool") + .unwrap() + .call1((value,)) + .unwrap() + .extract::() + .unwrap() + ); + }); + } + + #[rstest] + #[case("None", Ok(None), Ok(None), Ok(None))] + #[case("''", Ok(Some("")), Ok(None), Ok(None))] + #[case( + "' value '", + Ok(Some(" value ")), + Ok(Some(" value ")), + Ok(Some(" value ")) + )] + #[case("[]", Err(()), Ok(None), Ok(None))] + #[case("0", Err(()), Ok(None), Ok(None))] + #[case("1", Err(()), Err(()), Ok(None))] + #[case("object()", Err(()), Err(()), Ok(None))] + fn string_operations_do_not_conflate_absence_and_type_checks( + #[case] source: &str, + #[case] strict: Result, ()>, + #[case] fallback: Result, ()>, + #[case] tuning: Result, ()>, + ) { + Python::initialize(); + Python::attach(|py| { + let field = Field::new("test", "string", evaluate(py, source)); + let owned = + |expected: Result, ()>| expected.map(|value| value.map(str::to_owned)); + assert_eq!( + field.optional_strict_string().map_err(|_| ()), + owned(strict) + ); + assert_eq!( + field.falsy_optional_string().map_err(|_| ()), + owned(fallback) + ); + assert_eq!(field.tuning_string().map_err(|_| ()), owned(tuning)); + }); + } + + #[rstest] + #[case("None", None)] + #[case("' True '", Some(true))] + #[case("' fAlSe '", Some(false))] + #[case("'yes'", None)] + #[case("'1'", None)] + #[case("'unknown'", None)] + fn string_boolean_tokens_remain_separate_from_truthiness( + #[case] source: &str, + #[case] expected: Option, + ) { + Python::initialize(); + Python::attach(|py| { + assert_eq!( + Field::new("test", "flag", evaluate(py, source)) + .str_bool() + .unwrap(), + expected + ); + }); + } + + #[test] + fn protocol_errors_preserve_exception_identity_traceback_cause_and_context() { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c" +failure = LookupError('protocol failed') +cause = ValueError('cause') +context = RuntimeError('context') +def fail(): + try: + raise context + except RuntimeError: + raise failure from cause +class Bool: + def __bool__(self): return fail() +class Length: + def __len__(self): return fail() +class Iter: + def __iter__(self): return fail() +class Next: + def __iter__(self): return self + def __next__(self): return fail() +class Descriptor: + @property + def flag(self): return fail() +values = (Bool(), Length(), Iter(), Next(), [Bool()]) +descriptor = Descriptor() +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let values = locals.get_item("values").unwrap().unwrap(); + for value in values.try_iter().unwrap() { + let error = Field::new("test", "flag", value.unwrap()) + .string_collection() + .err() + .unwrap(); + let error = PyErr::from(error); + assert!( + error + .value(py) + .is(locals.get_item("failure").unwrap().unwrap()) + ); + assert!(error.is_instance_of::(py)); + assert!(error.traceback(py).is_some()); + assert!( + error + .value(py) + .getattr("__cause__") + .unwrap() + .is(locals.get_item("cause").unwrap().unwrap()) + ); + assert!( + error + .value(py) + .getattr("__context__") + .unwrap() + .is(locals.get_item("context").unwrap().unwrap()) + ); + } + let error = Field::read( + &locals.get_item("descriptor").unwrap().unwrap(), + "test", + "flag", + ) + .err() + .unwrap(); + assert!( + PyErr::from(error) + .value(py) + .is(locals.get_item("failure").unwrap().unwrap()) + ); + }); + } + + #[test] + fn identity_and_string_contents_do_not_invoke_unrelated_protocols() { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c" +class Hostile: + def __bool__(self): raise AssertionError('bool called') + def __eq__(self, other): raise AssertionError('eq called') + def __str__(self): raise AssertionError('str called') +class Text(str): + def __str__(self): raise AssertionError('str called') + def strip(self): raise AssertionError('strip called') + def lower(self): raise AssertionError('lower called') +hostile = Hostile() +text = Text(' False ') +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let hostile = Field::new("test", "flag", locals.get_item("hostile").unwrap().unwrap()); + assert!(!hostile.exact_true()); + assert!(matches!( + hostile.strict_string(), + Err(ProjectionError::InvalidConfiguration(_)) + )); + let text = Field::new("test", "flag", locals.get_item("text").unwrap().unwrap()); + assert_eq!(text.strict_string().unwrap(), " False "); + assert_eq!(text.str_bool().unwrap(), Some(false)); + }); + } + + #[test] + fn missing_snapshot_fields_and_descriptor_attribute_errors_are_distinct() { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c" +failure = AttributeError('descriptor failed') +class Snapshot: + @property + def flag(self): raise failure +snapshot = Snapshot() +class Dynamic: + def __getattr__(self, name): raise failure +class Intercepted: + def __getattribute__(self, name): raise failure +dynamic = Dynamic() +intercepted = Intercepted() +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let snapshot = locals.get_item("snapshot").unwrap().unwrap(); + let descriptor = PyErr::from(Field::read(&snapshot, "test", "flag").err().unwrap()); + assert!( + descriptor + .value(py) + .is(locals.get_item("failure").unwrap().unwrap()) + ); + for name in ["dynamic", "intercepted"] { + let value = locals.get_item(name).unwrap().unwrap(); + let error = PyErr::from(Field::read(&value, "test", "flag").err().unwrap()); + assert!( + error + .value(py) + .is(locals.get_item("failure").unwrap().unwrap()) + ); + } + let missing = PyErr::from(Field::read(&snapshot, "test", "missing").err().unwrap()); + assert!(missing.is_instance_of::(py)); + assert!(missing.to_string().contains("test.missing")); + }); + } + + #[test] + fn configuration_errors_name_fields_without_exposing_values() { + Python::initialize(); + Python::attach(|py| { + for source in [ + "{'secret': 'do-not-print'}", + "['host.test', {'secret': 'do-not-print'}]", + ] { + let field = Field::new("test", "setting", evaluate(py, source)); + let error = PyErr::from(field.falsy_optional_string().err().unwrap()); + assert!(error.is_instance_of::(py)); + assert!(error.to_string().contains("test.setting")); + assert!(!error.to_string().contains("do-not-print")); + } + let hosts = Field::new( + "url_policy", + "user_url_allowed_hosts", + evaluate(py, "['host.test', 1]"), + ); + assert!(matches!( + hosts.string_collection(), + Err(ProjectionError::InvalidConfiguration(_)) + )); + assert!(matches!( + Field::new("test", "flag", evaluate(py, "1")).str_bool(), + Err(ProjectionError::InvalidConfiguration(_)) + )); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/coercion/tests.rs b/litellm-rust/crates/python-bridge/src/coercion/tests.rs deleted file mode 100644 index 5ed237c3c64..00000000000 --- a/litellm-rust/crates/python-bridge/src/coercion/tests.rs +++ /dev/null @@ -1,372 +0,0 @@ -use std::ffi::CString; - -use pyo3::{ - exceptions::{PyLookupError, PyRuntimeError, PyValueError}, - types::PyDict, -}; -use rstest::rstest; - -use super::*; - -fn evaluate<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyAny> { - py.eval(&CString::new(source).unwrap(), None, None).unwrap() -} - -#[rstest] -#[case("None", false, false)] -#[case("False", false, false)] -#[case("True", true, true)] -#[case("0", false, false)] -#[case("1", true, false)] -#[case("''", false, false)] -#[case("'false'", true, false)] -#[case("[]", false, false)] -#[case("[0]", true, false)] -#[case("{}", false, false)] -#[case("object()", true, false)] -fn boolean_operations_have_distinct_python_semantics( - #[case] source: &str, - #[case] truth: bool, - #[case] exact: bool, -) { - Python::initialize(); - Python::attach(|py| { - let value = evaluate(py, source); - let field = Field::new("test.flag", value.clone()); - assert_eq!(field.truthy().unwrap().0, truth); - assert_eq!(field.exact_true().0, exact); - assert_eq!( - field.truthy().unwrap().0, - py.import("builtins") - .unwrap() - .getattr("bool") - .unwrap() - .call1((value,)) - .unwrap() - .extract::() - .unwrap() - ); - }); -} - -#[rstest] -#[case("None", Ok(None), Ok(None), Ok(None))] -#[case("''", Ok(Some("")), Ok(None), Ok(None))] -#[case( - "' value '", - Ok(Some(" value ")), - Ok(Some(" value ")), - Ok(Some(" value ")) -)] -#[case("[]", Err(()), Ok(None), Ok(None))] -#[case("0", Err(()), Ok(None), Ok(None))] -#[case("1", Err(()), Err(()), Ok(None))] -#[case("object()", Err(()), Err(()), Ok(None))] -fn string_operations_do_not_conflate_absence_and_type_checks( - #[case] source: &str, - #[case] strict: Result, ()>, - #[case] fallback: Result, ()>, - #[case] tuning: Result, ()>, -) { - Python::initialize(); - Python::attach(|py| { - let field = Field::new("test.string", evaluate(py, source)); - let owned = - |expected: Result, ()>| expected.map(|value| value.map(str::to_owned)); - assert_eq!( - field - .optional_strict_string() - .map(|value| value.0) - .map_err(|_| ()), - owned(strict) - ); - assert_eq!( - field - .falsy_optional_string() - .map(|value| value.0) - .map_err(|_| ()), - owned(fallback) - ); - assert_eq!( - field.tuning_string().map(|value| value.0).map_err(|_| ()), - owned(tuning) - ); - }); -} - -#[rstest] -#[case("None", None)] -#[case("' True '", Some(true))] -#[case("' fAlSe '", Some(false))] -#[case("'yes'", None)] -#[case("'1'", None)] -#[case("'unknown'", None)] -fn string_boolean_tokens_remain_separate_from_truthiness( - #[case] source: &str, - #[case] expected: Option, -) { - Python::initialize(); - Python::attach(|py| { - assert_eq!( - Field::new("test.flag", evaluate(py, source)) - .str_bool() - .unwrap() - .0, - expected - ); - }); -} - -#[rstest] -#[case("'EXAMPLE.TEST.'", vec!["example.test"])] -#[case("['B.test', '', None, 0, [], 'A.test.', 'b.test']", vec!["a.test", "b.test"])] -#[case("('B.test', 'a.test')", vec!["a.test", "b.test"])] -#[case("{'B.test', 'a.test'}", vec!["a.test", "b.test"])] -#[case("(host for host in ['B.test', 'a.test'])", vec!["a.test", "b.test"])] -#[case("None", vec![])] -#[case("False", vec![])] -fn host_collection_is_owned_normalized_and_deterministic( - #[case] source: &str, - #[case] expected: Vec<&str>, -) { - Python::initialize(); - Python::attach(|py| { - assert_eq!( - Field::new("url_policy.user_url_allowed_hosts", evaluate(py, source)) - .host_collection() - .unwrap() - .0, - expected - ); - }); -} - -#[test] -fn protocol_errors_preserve_exception_identity_traceback_cause_and_context() { - Python::initialize(); - Python::attach(|py| { - let locals = PyDict::new(py); - py.run( - c" -failure = LookupError('protocol failed') -cause = ValueError('cause') -context = RuntimeError('context') -def fail(): - try: - raise context - except RuntimeError: - raise failure from cause -class Bool: - def __bool__(self): return fail() -class Length: - def __len__(self): return fail() -class Iter: - def __iter__(self): return fail() -class Next: - def __iter__(self): return self - def __next__(self): return fail() -class Descriptor: - @property - def flag(self): return fail() -values = (Bool(), Length(), Iter(), Next(), [Bool()]) -descriptor = Descriptor() -", - Some(&locals), - Some(&locals), - ) - .unwrap(); - let values = locals.get_item("values").unwrap().unwrap(); - for value in values.try_iter().unwrap() { - let error = Field::new("test.flag", value.unwrap()) - .host_collection() - .err() - .unwrap(); - let error = PyErr::from(error); - assert!( - error - .value(py) - .is(locals.get_item("failure").unwrap().unwrap()) - ); - assert!(error.is_instance_of::(py)); - assert!(error.traceback(py).is_some()); - assert!( - error - .value(py) - .getattr("__cause__") - .unwrap() - .is(locals.get_item("cause").unwrap().unwrap()) - ); - assert!( - error - .value(py) - .getattr("__context__") - .unwrap() - .is(locals.get_item("context").unwrap().unwrap()) - ); - } - let error = Field::read( - &locals.get_item("descriptor").unwrap().unwrap(), - "test.flag", - ) - .err() - .unwrap(); - assert!( - PyErr::from(error) - .value(py) - .is(locals.get_item("failure").unwrap().unwrap()) - ); - }); -} - -#[test] -fn identity_and_string_contents_do_not_invoke_unrelated_protocols() { - Python::initialize(); - Python::attach(|py| { - let locals = PyDict::new(py); - py.run( - c" -class Hostile: - def __bool__(self): raise AssertionError('bool called') - def __eq__(self, other): raise AssertionError('eq called') - def __str__(self): raise AssertionError('str called') -class Text(str): - def __str__(self): raise AssertionError('str called') - def strip(self): raise AssertionError('strip called') - def lower(self): raise AssertionError('lower called') -hostile = Hostile() -text = Text(' False ') -", - Some(&locals), - Some(&locals), - ) - .unwrap(); - let hostile = Field::new("test.flag", locals.get_item("hostile").unwrap().unwrap()); - assert!(!hostile.exact_true().0); - assert!(matches!( - hostile.strict_string(), - Err(ProjectionError::InvalidConfiguration(_)) - )); - let text = Field::new("test.flag", locals.get_item("text").unwrap().unwrap()); - assert_eq!(text.strict_string().unwrap(), " False "); - assert_eq!(text.str_bool().unwrap().0, Some(false)); - }); -} - -#[test] -fn missing_snapshot_fields_and_descriptor_attribute_errors_are_distinct() { - Python::initialize(); - Python::attach(|py| { - let locals = PyDict::new(py); - py.run( - c" -failure = AttributeError('descriptor failed') -class Snapshot: - @property - def flag(self): raise failure -snapshot = Snapshot() -class Dynamic: - def __getattr__(self, name): raise failure -class Intercepted: - def __getattribute__(self, name): raise failure -dynamic = Dynamic() -intercepted = Intercepted() -", - Some(&locals), - Some(&locals), - ) - .unwrap(); - let snapshot = locals.get_item("snapshot").unwrap().unwrap(); - let descriptor = PyErr::from(Field::read(&snapshot, "test.flag").err().unwrap()); - assert!( - descriptor - .value(py) - .is(locals.get_item("failure").unwrap().unwrap()) - ); - for name in ["dynamic", "intercepted"] { - let value = locals.get_item(name).unwrap().unwrap(); - let error = PyErr::from(Field::read(&value, "test.flag").err().unwrap()); - assert!( - error - .value(py) - .is(locals.get_item("failure").unwrap().unwrap()) - ); - } - let missing = PyErr::from(Field::read(&snapshot, "test.missing").err().unwrap()); - assert!(missing.is_instance_of::(py)); - assert!(missing.to_string().contains("test.missing")); - }); -} - -#[test] -fn configuration_errors_name_fields_without_exposing_values() { - Python::initialize(); - Python::attach(|py| { - for source in [ - "{'secret': 'do-not-print'}", - "['host.test', {'secret': 'do-not-print'}]", - ] { - let field = Field::new("test.setting", evaluate(py, source)); - let error = PyErr::from(field.falsy_optional_string().err().unwrap()); - assert!(error.is_instance_of::(py)); - assert!(error.to_string().contains("test.setting")); - assert!(!error.to_string().contains("do-not-print")); - } - let hosts = Field::new( - "url_policy.user_url_allowed_hosts", - evaluate(py, "['host.test', 1]"), - ); - assert!(matches!( - hosts.host_collection(), - Err(ProjectionError::InvalidConfiguration(_)) - )); - assert!(matches!( - Field::new("test.flag", evaluate(py, "1")).str_bool(), - Err(ProjectionError::InvalidConfiguration(_)) - )); - }); -} - -#[test] -fn projection_releases_the_source_collection() { - Python::initialize(); - Python::attach(|py| { - let source = evaluate(py, "['A.test']"); - let projected = Field::new("test.hosts", source.clone()) - .host_collection() - .unwrap() - .0; - source.call_method1("append", ("b.test",)).unwrap(); - assert_eq!(projected, ["a.test"]); - assert_eq!( - Field::new("test.hosts", source) - .host_collection() - .unwrap() - .0, - ["a.test", "b.test"] - ); - }); -} - -#[rstest] -#[case("True", Some(true))] -#[case("False", Some(false))] -#[case("1", None)] -#[case("None", None)] -#[case("[]", None)] -fn accessor_booleans_are_strict_schema_values( - #[case] source: &str, - #[case] expected: Option, -) { - Python::initialize(); - Python::attach(|py| { - let result = Field::new("secret_manager.readable", evaluate(py, source)).schema_bool(); - match expected { - Some(expected) => assert_eq!(result.unwrap(), expected), - None => { - let error = PyErr::from(result.unwrap_err()); - assert!(error.is_instance_of::(py)); - assert!(error.to_string().contains("secret_manager.readable")); - } - } - }); -} diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index 596a89a73d7..2515b409c54 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -1,5 +1,5 @@ use std::{ - collections::HashSet, + collections::{BTreeSet, HashSet}, path::{Path, PathBuf}, sync::{Arc, LazyLock, Mutex, PoisonError}, }; @@ -10,9 +10,75 @@ use litellm_http::{ TlsSource, Unsupported, media::{PublicDnsResolver, UrlPolicy}, }; -use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; +use pyo3::{ + exceptions::PyValueError, + prelude::*, + types::{PyBool, PyDict, PyString}, +}; -use crate::{coercion::Field, python_settings::PythonSettings}; +use crate::{ + coercion::{Field, FieldSpec, ProjectionError}, + python_settings::{PythonSettings, Snapshot}, +}; + +const SSL_VERIFY: FieldSpec> = FieldSpec::new("ssl_verify", decode_ssl_verify); +const SSL_CERTIFICATE: FieldSpec> = + FieldSpec::new("ssl_certificate", |field| field.optional_strict_string()); +const SSL_SECURITY_LEVEL: FieldSpec> = + FieldSpec::new("ssl_security_level", |field| field.tuning_string()); +const SSL_ECDH_CURVE: FieldSpec> = + FieldSpec::new("ssl_ecdh_curve", |field| field.tuning_string()); +const FORCE_IPV4: FieldSpec = FieldSpec::new("force_ipv4", |field| field.truthy()); +const HTTP2: FieldSpec = FieldSpec::new("http2", |field| Ok(field.exact_true())); +const AIOHTTP_TRUST_ENV: FieldSpec = + FieldSpec::new("aiohttp_trust_env", |field| field.truthy()); +const DISABLE_AIOHTTP_TRUST_ENV: FieldSpec = + FieldSpec::new("disable_aiohttp_trust_env", |field| field.truthy()); +const DISABLE_AIOHTTP_TRANSPORT: FieldSpec = + FieldSpec::new("disable_aiohttp_transport", |field| Ok(field.exact_true())); +const USER_AGENT: FieldSpec = FieldSpec::new("user_agent", |field| field.schema_string()); +const USER_URL_VALIDATION: FieldSpec = + FieldSpec::new("user_url_validation", |field| field.truthy()); +const USER_URL_ALLOWED_HOSTS: FieldSpec> = + FieldSpec::new("user_url_allowed_hosts", decode_hosts); + +fn decode_hosts(field: &Field<'_>) -> Result, ProjectionError> { + Ok(field + .string_collection()? + .into_iter() + .map(|host| litellm_http::media::normalize_host(&host)) + .collect::>() + .into_iter() + .collect()) +} + +fn decode_ssl_verify(field: &Field<'_>) -> Result, ProjectionError> { + let value = field.value(); + if value.is_none() { + return Ok(None); + } + if value.is_instance_of::() { + return Ok(Some(if field.exact_true() { + SslVerify::Enabled + } else { + SslVerify::Disabled + })); + } + if value.is_instance_of::() { + return Ok(Some(match field.str_bool()? { + Some(true) => SslVerify::Enabled, + Some(false) => SslVerify::Disabled, + None => SslVerify::CaBundle(field.strict_string()?.into()), + })); + } + let context = value.py().import("ssl")?.getattr("SSLContext")?; + if value.is_instance(&context)? { + return Err(ProjectionError::UnsupportedLiveObject(field.expected( + "a Boolean, Boolean string, CA path, or None; live SSLContext is unsupported", + )?)); + } + Err(field.invalid("a Boolean, Boolean string, CA path, or None")) +} static POOL: LazyLock = LazyLock::new(|| HttpClientPool::new(Arc::new(PublicDnsResolver))); @@ -80,20 +146,20 @@ pub(crate) fn url_policy(py: Python<'_>) -> PyResult { project_url_policy(&PythonSettings::UrlPolicy.read(py)?) } -fn project_url_policy(value: &Bound<'_, PyAny>) -> PyResult { +fn project_url_policy(snapshot: &Snapshot<'_>) -> PyResult { Ok(UrlPolicy { - validate: Field::read(value, "url_policy.user_url_validation")? - .truthy()? - .0, - allowed_hosts: Field::read(value, "url_policy.user_url_allowed_hosts")? - .host_collection()? - .0, + validate: snapshot.read(&USER_URL_VALIDATION)?, + allowed_hosts: snapshot.read(&USER_URL_ALLOWED_HOSTS)?, }) } fn call_ssl_verify(kwargs: &Bound<'_, PyDict>) -> PyResult> { match kwargs.get_item("ssl_verify")? { - Some(value) => Ok(Field::new("request.ssl_verify", value).ssl_verify()?.0), + Some(value) => Ok(decode_ssl_verify(&Field::new( + "request", + "ssl_verify", + value, + ))?), None => Ok(None), } } @@ -106,39 +172,18 @@ fn for_call(call_ssl_verify: Option, asynchronous: bool) -> HttpSetti } } -fn configured(value: &Bound<'_, PyAny>) -> PyResult { +fn configured(snapshot: &Snapshot<'_>) -> PyResult { Ok(HttpSettingsLayer { - ssl_verify: Field::read(value, "http_settings.ssl_verify")? - .ssl_verify()? - .0, - ssl_certificate: Field::read(value, "http_settings.ssl_certificate")? - .optional_strict_string()? - .0 - .map(PathBuf::from), - ssl_security_level: Field::read(value, "http_settings.ssl_security_level")? - .tuning_string()? - .0, - ssl_ecdh_curve: Field::read(value, "http_settings.ssl_ecdh_curve")? - .tuning_string()? - .0, - force_ipv4: Some(Field::read(value, "http_settings.force_ipv4")?.truthy()?.0), - http2: Some(Field::read(value, "http_settings.http2")?.exact_true().0), - aiohttp_trust_env: Some( - Field::read(value, "http_settings.aiohttp_trust_env")? - .truthy()? - .0, - ), - disable_aiohttp_trust_env: Some( - Field::read(value, "http_settings.disable_aiohttp_trust_env")? - .truthy()? - .0, - ), - disable_aiohttp_transport: Some( - Field::read(value, "http_settings.disable_aiohttp_transport")? - .exact_true() - .0, - ), - user_agent: Some(Field::read(value, "http_settings.user_agent")?.schema_string()?), + ssl_verify: snapshot.read(&SSL_VERIFY)?, + ssl_certificate: snapshot.read(&SSL_CERTIFICATE)?.map(PathBuf::from), + ssl_security_level: snapshot.read(&SSL_SECURITY_LEVEL)?, + ssl_ecdh_curve: snapshot.read(&SSL_ECDH_CURVE)?, + force_ipv4: Some(snapshot.read(&FORCE_IPV4)?), + http2: Some(snapshot.read(&HTTP2)?), + aiohttp_trust_env: Some(snapshot.read(&AIOHTTP_TRUST_ENV)?), + disable_aiohttp_trust_env: Some(snapshot.read(&DISABLE_AIOHTTP_TRUST_ENV)?), + disable_aiohttp_transport: Some(snapshot.read(&DISABLE_AIOHTTP_TRANSPORT)?), + user_agent: Some(snapshot.read(&USER_AGENT)?), ..HttpSettingsLayer::default() }) } @@ -150,12 +195,15 @@ mod tests { use rstest::rstest; use super::*; - use crate::python_settings::CONTRACT; - fn python_settings<'py>(py: Python<'py>, overrides: &str) -> Bound<'py, PyAny> { + fn evaluate<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyAny> { + py.eval(&std::ffi::CString::new(source).unwrap(), None, None) + .unwrap() + } + + fn python_settings<'py>(py: Python<'py>, overrides: &str) -> Snapshot<'py> { let source = format!( " -import json import types defaults = dict( ssl_verify=True, @@ -170,14 +218,13 @@ defaults = dict( user_agent='litellm/test', ) defaults.update(dict({overrides})) -settings = types.SimpleNamespace(**{{name: defaults[name] for name in json.loads(contract)['http_settings']['fields']}}) +settings = types.SimpleNamespace(**defaults) " ); let locals = PyDict::new(py); - locals.set_item("contract", CONTRACT).unwrap(); let source = std::ffi::CString::new(source).unwrap(); py.run(&source, Some(&locals), Some(&locals)).unwrap(); - locals.get_item("settings").unwrap().unwrap() + PythonSettings::Http.snapshot(locals.get_item("settings").unwrap().unwrap()) } #[test] @@ -395,7 +442,7 @@ user_agent='litellm/9.9.9', Python::attach(|py| { let value = py.eval(c"__import__('types').SimpleNamespace(user_url_validation=[], user_url_allowed_hosts=['B.test', 'a.test.', 'b.test'])", None, None).unwrap(); assert_eq!( - project_url_policy(&value).unwrap(), + project_url_policy(&PythonSettings::UrlPolicy.snapshot(value)).unwrap(), UrlPolicy { validate: false, allowed_hosts: vec!["a.test".into(), "b.test".into()], @@ -419,4 +466,44 @@ user_agent='litellm/9.9.9', let settings = HttpSettings::from_layers([for_call(None, asynchronous), opted_out]); assert_eq!(settings.trust_proxy_env, expected); } + #[rstest] + #[case("'EXAMPLE.TEST.'", vec!["example.test"])] + #[case("['B.test', '', None, 0, [], 'A.test.', 'b.test']", vec!["a.test", "b.test"])] + #[case("('B.test', 'a.test')", vec!["a.test", "b.test"])] + #[case("{'B.test', 'a.test'}", vec!["a.test", "b.test"])] + #[case("(host for host in ['B.test', 'a.test'])", vec!["a.test", "b.test"])] + #[case("None", vec![])] + #[case("False", vec![])] + fn host_collection_is_owned_normalized_and_deterministic( + #[case] source: &str, + #[case] expected: Vec<&str>, + ) { + Python::initialize(); + Python::attach(|py| { + assert_eq!( + decode_hosts(&Field::new( + "url_policy", + "user_url_allowed_hosts", + evaluate(py, source) + )) + .unwrap(), + expected + ); + }); + } + + #[test] + fn projection_releases_the_source_collection() { + Python::initialize(); + Python::attach(|py| { + let source = evaluate(py, "['A.test']"); + let projected = decode_hosts(&Field::new("test", "hosts", source.clone())).unwrap(); + source.call_method1("append", ("b.test",)).unwrap(); + assert_eq!(projected, ["a.test"]); + assert_eq!( + decode_hosts(&Field::new("test", "hosts", source)).unwrap(), + ["a.test", "b.test"] + ); + }); + } } diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index f13a3ad433f..b1fc5244d6f 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -7,7 +7,12 @@ mod http; mod marshal; mod python_settings; mod routes; -mod token_counter; +#[allow( + dead_code, + reason = "secret-manager foundations await rollout activation" +)] +mod secrets; +mod tokenizer; #[pymodule(gil_used = true)] mod _native { @@ -32,7 +37,12 @@ mod _native { #[pymodule_export] use crate::routes::responses::ResponsesWebSocketConnection; #[pymodule_export] - use crate::token_counter::TokenCounter; + use crate::routes::token_counter::TokenCounter; + #[cfg(feature = "huggingface")] + #[pymodule_export] + use crate::tokenizer::HuggingFaceEncoding; + #[pymodule_export] + use crate::tokenizer::Tokenizer; #[pymodule_export] use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking}; use pyo3::{prelude::*, types::PyModule}; @@ -43,7 +53,7 @@ mod _native { let dict = module.dict(); dict.set_item("_CacheTestHandle", py.get_type::())?; dict.set_item("_CacheTestResolver", py.get_type::())?; - dict.set_item("_CacheTestBinding", py.get_type::()) + dict.set_item("_ResponseCacheRuntime", py.get_type::()) } } @@ -78,10 +88,13 @@ mod tests { "achat_completions", "ResponsesWebSocketConnection", "TokenCounter", + "Tokenizer", "gil_stats", "process_state_started", "reserve_process_for_forking", ]; + #[cfg(feature = "huggingface")] + expected.push("HuggingFaceEncoding"); expected.sort_unstable(); let mut public_names: Vec = native_module(py) diff --git a/litellm-rust/crates/python-bridge/src/python_settings.rs b/litellm-rust/crates/python-bridge/src/python_settings.rs index bdc6d14356d..111ac3bc259 100644 --- a/litellm-rust/crates/python-bridge/src/python_settings.rs +++ b/litellm-rust/crates/python-bridge/src/python_settings.rs @@ -1,5 +1,7 @@ use pyo3::prelude::*; +use crate::coercion::{FieldSpec, ProjectionError}; + const MODULE: &str = "litellm.rust_bridge.settings"; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -8,28 +10,39 @@ pub(crate) enum PythonSettings { UrlPolicy, ProviderDefaults, SecretManager, + SecretManagerBinding, +} + +pub(crate) struct Snapshot<'py> { + group: PythonSettings, + value: Bound<'py, PyAny>, +} + +impl Snapshot<'_> { + pub(crate) fn read(&self, spec: &FieldSpec) -> Result { + spec.read(&self.value, self.group.name()) + } } impl PythonSettings { - #[cfg(test)] - pub(crate) const ALL: [Self; 4] = [ - Self::Http, - Self::UrlPolicy, - Self::ProviderDefaults, - Self::SecretManager, - ]; - pub(crate) fn name(self) -> &'static str { match self { Self::Http => "http_settings", Self::UrlPolicy => "url_policy", Self::ProviderDefaults => "provider_defaults", Self::SecretManager => "secret_manager", + Self::SecretManagerBinding => "secret_manager_binding", } } - pub(crate) fn read(self, py: Python<'_>) -> PyResult> { - py.import(MODULE)?.getattr(self.name())?.call0() + pub(crate) fn read(self, py: Python<'_>) -> PyResult> { + let value = py.import(MODULE)?.getattr(self.name())?.call0()?; + Ok(Snapshot { group: self, value }) + } + + #[cfg(test)] + pub(crate) fn snapshot(self, value: Bound<'_, PyAny>) -> Snapshot<'_> { + Snapshot { group: self, value } } pub(crate) fn warn(py: Python<'_>, message: &str) -> PyResult<()> { @@ -38,209 +51,98 @@ impl PythonSettings { } } -#[cfg(test)] -pub(crate) const CONTRACT: &str = include_str!("../python_settings.json"); - #[cfg(test)] mod tests { - use super::{CONTRACT, PythonSettings}; - use pyo3::prelude::*; - use serde_json::{Value, json}; + use pyo3::{exceptions::PyRuntimeError, prelude::*, types::PyDict}; - struct SettingSpec { - group: &'static str, - name: &'static str, - adapter: &'static str, - precedence: &'static str, - sensitive: bool, - shapes: &'static [&'static str], - unsupported_live: Option<&'static str>, - } - - const SETTINGS: &[SettingSpec] = &[ - SettingSpec { - group: "http_settings", - name: "ssl_verify", - adapter: "SslVerifyInput", - precedence: "module_global", - sensitive: false, - shapes: &["none", "bool", "str"], - unsupported_live: Some("configuration_error"), - }, - SettingSpec { - group: "http_settings", - name: "ssl_certificate", - adapter: "OptionalStrictString", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "http_settings", - name: "ssl_security_level", - adapter: "TuningString", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "http_settings", - name: "ssl_ecdh_curve", - adapter: "TuningString", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "http_settings", - name: "force_ipv4", - adapter: "Truthy", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "http_settings", - name: "http2", - adapter: "ExactTrue", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "http_settings", - name: "aiohttp_trust_env", - adapter: "Truthy", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "http_settings", - name: "disable_aiohttp_trust_env", - adapter: "Truthy", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "http_settings", - name: "disable_aiohttp_transport", - adapter: "ExactTrue", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "http_settings", - name: "user_agent", - adapter: "StrictString", - precedence: "accessor", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "url_policy", - name: "user_url_validation", - adapter: "Truthy", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "url_policy", - name: "user_url_allowed_hosts", - adapter: "HostCollection", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "provider_defaults", - name: "vertex_project", - adapter: "FalsyOptionalString", - precedence: "module_global", - sensitive: true, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "provider_defaults", - name: "vertex_location", - adapter: "FalsyOptionalString", - precedence: "module_global", - sensitive: true, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "provider_defaults", - name: "enable_azure_ad_token_refresh", - adapter: "ExactTrue", - precedence: "module_global", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - SettingSpec { - group: "secret_manager", - name: "readable", - adapter: "StrictBool", - precedence: "accessor", - sensitive: false, - shapes: &[], - unsupported_live: None, - }, - ]; + use super::PythonSettings; + use crate::coercion::FieldSpec; #[test] - fn settings_manifest_matches_the_semantic_contract() { - pyo3::Python::initialize(); - let manifest: Value = pyo3::Python::attach(|py| { - let value = py - .import("json") - .unwrap() - .call_method1("loads", (CONTRACT,)) - .unwrap(); - litellm_host_python::from_py(&value).unwrap() + fn declarations_select_the_decoder_and_read_only_the_requested_field() { + const TRUTHY: FieldSpec = FieldSpec::new("flag", |field| field.truthy()); + const EXACT: FieldSpec = FieldSpec::new("flag", |field| Ok(field.exact_true())); + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c" +reads = [] +class Settings: + value = 1 + @property + def flag(self): + reads.append('flag') + return self.value + @property + def unrelated(self): + raise AssertionError('unrequested field') +settings = Settings() +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let value = locals.get_item("settings").unwrap().unwrap(); + let snapshot = PythonSettings::Http.snapshot(value.clone()); + assert!(snapshot.read(&TRUTHY).unwrap()); + assert!(!snapshot.read(&EXACT).unwrap()); + value.setattr("value", true).unwrap(); + assert!(snapshot.read(&EXACT).unwrap()); + assert_eq!( + locals + .get_item("reads") + .unwrap() + .unwrap() + .extract::>() + .unwrap(), + ["flag", "flag", "flag"] + ); + }); + } + + #[test] + fn declared_reads_preserve_descriptor_and_decoder_failures_and_name_missing_fields() { + const FLAG: FieldSpec = FieldSpec::new("flag", |field| field.truthy()); + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c" +from types import SimpleNamespace +failure = AttributeError('read failed') +class Descriptor: + @property + def flag(self): raise failure +class Truth: + def __bool__(self): raise failure +values = (Descriptor(), SimpleNamespace(flag=Truth())) +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let failure = locals.get_item("failure").unwrap().unwrap(); + for value in locals + .get_item("values") + .unwrap() + .unwrap() + .try_iter() + .unwrap() + { + let snapshot = PythonSettings::Http.snapshot(value.unwrap()); + let error = PyErr::from(snapshot.read(&FLAG).unwrap_err()); + assert!(error.value(py).is(&failure)); + assert!(error.traceback(py).is_some()); + } + let missing = PythonSettings::Http.snapshot(py.eval(c"object()", None, None).unwrap()); + let error = PyErr::from(missing.read(&FLAG).unwrap_err()); + assert!(error.is_instance_of::(py)); + assert!( + error + .to_string() + .contains("http_settings.flag: missing snapshot field") + ); }); - let expected: serde_json::Map = PythonSettings::ALL - .into_iter() - .map(|group| { - let fields: serde_json::Map = SETTINGS - .iter() - .filter(|spec| spec.group == group.name()) - .map(|spec| { - ( - spec.name.to_owned(), - json!({ - "adapter": spec.adapter, - "required": true, - "precedence": spec.precedence, - "sensitive": spec.sensitive, - "shapes": spec.shapes, - "unsupported_live": spec.unsupported_live, - }), - ) - }) - .collect(); - ( - group.name().to_owned(), - json!({"version": 1, "fields": fields}), - ) - }) - .collect(); - assert_eq!(manifest, Value::Object(expected)); } } diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index 2d6b849a6b1..8a78a26423d 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod chat_completions; pub(crate) mod messages; pub(crate) mod ocr; pub(crate) mod responses; +pub(crate) mod token_counter; #[cfg(test)] mod tests { diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs index 77c8d5d6641..8bf99cd355f 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs @@ -130,6 +130,11 @@ impl RouteHost for OcrRouteHost { } fn classify(&self, py: Python<'_>, error: Error) -> PyResult { + if let Error::Secret(source) = &error + && let Some(original) = crate::secrets::python_error(py, source) + { + return Ok(original); + } Ok(self.map_failure(py, ocr_error_to_pyerr(error))) } diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index d0b13e5056a..2dca6da66cd 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -10,16 +10,33 @@ use litellm_auth_gcp::VertexAuth; use litellm_callbacks_legacy_python::{LegacySurface, PublicCall, run_legacy_call}; use litellm_core::ocr::route::ocr_machine; use litellm_core_utils::settings::ProcessEnvironment; -use litellm_llms::base_llm::ocr::{ - handler::OcrClient, - settings::{OcrSettings, Secrets}, +use litellm_llms::base_llm::{ + inference::secrets::{EnvironmentSecrets, SecretSource}, + ocr::{handler::OcrClient, settings::OcrSettings}, }; use pyo3::{ prelude::*, types::{PyDict, PyTuple}, }; -use crate::{coercion::Field, errors::RustBridgeDeclined, http, python_settings::PythonSettings}; +use crate::{ + coercion::FieldSpec, + errors::RustBridgeDeclined, + http, + python_settings::{PythonSettings, Snapshot}, +}; + +const SECRET_MANAGER_READABLE: FieldSpec = + FieldSpec::new("readable", |field| field.schema_bool()); + +const VERTEX_PROJECT: FieldSpec> = + FieldSpec::new("vertex_project", |field| field.falsy_optional_string()); +const VERTEX_LOCATION: FieldSpec> = + FieldSpec::new("vertex_location", |field| field.falsy_optional_string()); +const ENABLE_AZURE_AD_TOKEN_REFRESH: FieldSpec = + FieldSpec::new("enable_azure_ad_token_refresh", |field| { + Ok(field.exact_true()) + }); const SURFACE: LegacySurface = LegacySurface { call_type: "ocr", @@ -62,33 +79,24 @@ fn run_ocr( ) } -fn process_environment_secrets(secret_manager: &Bound<'_, PyAny>) -> PyResult { - if Field::read(secret_manager, "secret_manager.readable")?.schema_bool()? { +fn process_environment_secrets(snapshot: &Snapshot<'_>) -> PyResult> { + if snapshot.read(&SECRET_MANAGER_READABLE)? { return Err(RustBridgeDeclined::new_err( "a readable secret manager is configured and the Rust route only reads the process environment", )); } - Ok(Arc::new(ProcessEnvironment)) + Ok(Arc::new(EnvironmentSecrets)) } fn ocr_settings(py: Python<'_>) -> PyResult { project_provider_defaults(&PythonSettings::ProviderDefaults.read(py)?) } -fn project_provider_defaults(value: &Bound<'_, PyAny>) -> PyResult { +fn project_provider_defaults(snapshot: &Snapshot<'_>) -> PyResult { Ok(OcrSettings { - vertex_project: Field::read(value, "provider_defaults.vertex_project")? - .falsy_optional_string()? - .0, - vertex_location: Field::read(value, "provider_defaults.vertex_location")? - .falsy_optional_string()? - .0, - enable_azure_ad_token_refresh: Field::read( - value, - "provider_defaults.enable_azure_ad_token_refresh", - )? - .exact_true() - .0, + vertex_project: snapshot.read(&VERTEX_PROJECT)?, + vertex_location: snapshot.read(&VERTEX_LOCATION)?, + enable_azure_ad_token_refresh: snapshot.read(&ENABLE_AZURE_AD_TOKEN_REFRESH)?, ..OcrSettings::from_environment(&ProcessEnvironment) }) } @@ -120,6 +128,8 @@ mod tests { use super::process_environment_secrets; use crate::errors::RustBridgeDeclined; + use crate::python_settings::PythonSettings; + fn secret_manager<'py>(py: Python<'py>, readable: bool) -> Bound<'py, PyAny> { let locals = PyDict::new(py); locals.set_item("readable", readable).unwrap(); @@ -132,12 +142,26 @@ mod tests { locals.get_item("manager").unwrap().unwrap() } + #[test] + fn a_readable_secret_manager_sends_the_call_back_to_python() { + Python::initialize(); + Python::attach(|py| { + let declined = process_environment_secrets( + &PythonSettings::SecretManager.snapshot(secret_manager(py, true)), + ) + .err() + .expect("the Rust route declines"); + assert!(declined.is_instance_of::(py)); + }); + } + #[test] fn provider_defaults_distinguish_falsey_values_and_exact_true() { Python::initialize(); Python::attach(|py| { let value = py.eval(c"__import__('types').SimpleNamespace(vertex_project=[], vertex_location=0, enable_azure_ad_token_refresh=1)", None, None).unwrap(); - let projected = super::project_provider_defaults(&value).unwrap(); + let snapshot = PythonSettings::ProviderDefaults.snapshot(value.clone()); + let projected = super::project_provider_defaults(&snapshot).unwrap(); assert_eq!(projected.vertex_project, None); assert_eq!(projected.vertex_location, None); assert!(!projected.enable_azure_ad_token_refresh); @@ -146,12 +170,12 @@ mod tests { value .setattr("enable_azure_ad_token_refresh", true) .unwrap(); - let next = super::project_provider_defaults(&value).unwrap(); + let next = super::project_provider_defaults(&snapshot).unwrap(); assert_eq!(next.vertex_project.as_deref(), Some("project")); assert_eq!(next.vertex_location.as_deref(), Some("region")); assert!(next.enable_azure_ad_token_refresh); value.setattr("vertex_project", 1).unwrap(); - let error = super::project_provider_defaults(&value).err().unwrap(); + let error = super::project_provider_defaults(&snapshot).err().unwrap(); assert!(error.is_instance_of::(py)); assert!( error @@ -160,28 +184,4 @@ mod tests { ); }); } - - #[test] - fn a_readable_secret_manager_sends_the_call_back_to_python() { - Python::initialize(); - Python::attach(|py| { - let declined = process_environment_secrets(&secret_manager(py, true)) - .err() - .expect("the Rust route declines"); - assert!(declined.is_instance_of::(py)); - }); - } - - #[test] - fn without_a_readable_secret_manager_secrets_are_the_process_environment() { - Python::initialize(); - Python::attach(|py| { - let secrets = process_environment_secrets(&secret_manager(py, false)).unwrap(); - assert_eq!( - secrets.get("LITELLM_RUST_BRIDGE_UNSET_VARIABLE_FOR_TEST"), - None - ); - assert_eq!(secrets.get("PATH"), std::env::var("PATH").ok()); - }); - } } diff --git a/litellm-rust/crates/python-bridge/src/routes/token_counter.rs b/litellm-rust/crates/python-bridge/src/routes/token_counter.rs new file mode 100644 index 00000000000..168c4883b0b --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/token_counter.rs @@ -0,0 +1,87 @@ +use std::sync::Arc; +use std::{num::NonZero, thread::available_parallelism}; + +use litellm_host_python::{enter_native, run_async}; +use litellm_token_counter::{ + CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter, +}; +use pyo3::{ + exceptions::{PyRuntimeError, PyValueError}, + prelude::*, + types::PyAny, +}; +use tokio::sync::Semaphore; + +use crate::errors::RustBridgeDeclined; +use crate::tokenizer::Tokenizer; + +/// Counts the input tokens of a raw request body off the Python event loop with +/// the GIL released. Python owns which requests get here and what to do with +/// the count. At most one encode per core runs at a time; the rest wait in the +/// async task, where a cancelled Python awaiter drops them before any blocking +/// work is scheduled. +#[pyclass(frozen)] +pub(crate) struct TokenCounter { + inner: Arc, + encode_slots: Arc, +} + +#[pymethods] +impl TokenCounter { + #[staticmethod] + #[pyo3(signature = (tokenizer, fast = false))] + fn from_tokenizer(py: Python<'_>, tokenizer: &Tokenizer, fast: bool) -> PyResult { + enter_native()?; + let inner = CoreTokenCounter::new(tokenizer.counter(py, fast)); + Ok(Self { + inner: Arc::new(inner), + encode_slots: Arc::new(Semaphore::new(encode_parallelism())), + }) + } + + fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult> { + let counter = Arc::clone(&self.inner); + let encode_slots = Arc::clone(&self.encode_slots); + let body = body.to_vec(); + run_async( + py, + async move { + let _slot = encode_slots + .acquire_owned() + .await + .map_err(|error| Error::Task(error.to_string()))?; + tokio::task::spawn_blocking(move || count_body(&counter, &body)) + .await + .map_err(|error| Error::Task(error.to_string()))? + }, + token_count_error_to_pyerr, + ) + } +} + +fn encode_parallelism() -> usize { + available_parallelism().map_or(1, NonZero::get) +} + +fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result { + let request = CountableRequest::parse(body)?; + counter.count_request(&request) +} + +pub(crate) fn token_count_error_to_pyerr(error: Error) -> PyErr { + let message = error.to_string(); + match error { + Error::Load(_) + | Error::Ranks(_) + | Error::UnicodeClasses + | Error::UnsupportedTokenizer(_) => PyValueError::new_err(message), + Error::RequestParse(_) + | Error::MissingInput + | Error::FloatText + | Error::ContentBlock + | Error::ArrayItems + | Error::JsonSerialization(_) + | Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message), + Error::Encode(_) | Error::Decode(_) | Error::Task(_) => PyRuntimeError::new_err(message), + } +} diff --git a/litellm-rust/crates/python-bridge/src/secrets/callback.rs b/litellm-rust/crates/python-bridge/src/secrets/callback.rs new file mode 100644 index 00000000000..a8e5850fe3f --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/secrets/callback.rs @@ -0,0 +1,352 @@ +use std::{future::Future, pin::Pin}; + +use litellm_core_utils::settings::Lookup; +use litellm_secrets::{ + Error, ExternalSecretManager, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue, +}; +use pyo3::{prelude::*, types::PyDict}; + +use super::error::external_error; + +const HANDLER_MODULE: &str = "litellm.secret_managers.secret_manager_handler"; + +/// A secret manager whose reads execute in Python: a custom manager, a legacy compatible +/// client, or a manually assigned SDK client. +pub(crate) struct PythonSecretManager { + client: Py, + system: Option, + settings: Option>, +} + +impl PythonSecretManager { + pub(crate) fn new( + client: Py, + system: Option, + settings: Option>, + ) -> Self { + Self { + client, + system, + settings, + } + } + + fn read(&self, py: Python<'_>, name: &str) -> PyResult> { + let client = self.client.bind(py); + if self.system == Some(KeyManagementSystem::Custom) + || (self.system.is_none() && client.hasattr("sync_read_secret")?) + { + let kwargs = PyDict::new(py); + kwargs.set_item("secret_name", name)?; + if self.system == Some(KeyManagementSystem::Custom) { + let optional_params = self + .settings + .as_ref() + .map(|settings| settings.bind(py).call_method0("model_dump")) + .transpose()?; + kwargs.set_item("optional_params", optional_params)?; + } + return client + .call_method("sync_read_secret", (), Some(&kwargs))? + .extract(); + } + let kwargs = PyDict::new(py); + kwargs.set_item("client", client)?; + kwargs.set_item("key_manager", self.system.map_or("local", python_name))?; + kwargs.set_item("secret_name", name)?; + match &self.settings { + Some(settings) => kwargs.set_item("key_management_settings", settings.bind(py))?, + None => kwargs.set_item("key_management_settings", py.None())?, + } + py.import(HANDLER_MODULE)? + .getattr("get_secret_from_manager")? + .call((), Some(&kwargs))? + .extract() + } +} + +/// The `KeyManagementSystem` value as Python spells it. +fn python_name(system: KeyManagementSystem) -> &'static str { + match system { + KeyManagementSystem::GoogleKms => "google_kms", + KeyManagementSystem::AzureKeyVault => "azure_key_vault", + KeyManagementSystem::AwsSecretManager => "aws_secret_manager", + KeyManagementSystem::GoogleSecretManager => "google_secret_manager", + KeyManagementSystem::HashicorpVault => "hashicorp_vault", + KeyManagementSystem::Cyberark => "cyberark", + KeyManagementSystem::Local => "local", + KeyManagementSystem::AwsKms => "aws_kms", + KeyManagementSystem::Custom => "custom", + } +} + +impl ExternalSecretManager for PythonSecretManager { + fn system(&self) -> KeyManagementSystem { + self.system.unwrap_or(KeyManagementSystem::Custom) + } + + fn read_secret<'a>( + &'a self, + name: &'a str, + _settings: &'a KeyManagementSettings, + _environment: &'a (dyn Lookup + Send + Sync), + ) -> Pin, Error>> + Send + 'a>> { + Box::pin(async move { + Python::attach(|py| { + self.read(py, name) + .map(|value| value.map(SecretValue::new).map(Secret::String)) + .map_err(|error| external_error(py, error)) + }) + }) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use litellm_secrets::{ + FailurePolicy, KeyManagementSettings, KeyManagementSystem, OidcResolver, SecretManager, + SecretManagerState, SecretResolver, + }; + use pyo3::{prelude::*, types::PyDict}; + use rstest::rstest; + + use super::{HANDLER_MODULE, PythonSecretManager, python_name}; + use crate::secrets::python_error; + + #[rstest] + #[case::value_error("ValueError", None)] + #[case::value_error_with_fallback("ValueError", Some("environment-key"))] + #[case::cancelled("asyncio.CancelledError", None)] + #[case::cancelled_with_fallback("asyncio.CancelledError", Some("environment-key"))] + #[tokio::test] + async fn callback_failures_preserve_python_exceptions_even_with_environment_fallback( + #[case] failure_type: &str, + #[case] fallback: Option<&'static str>, + ) { + Python::initialize(); + let (reader, locals) = Python::attach(|py| { + let locals = PyDict::new(py); + locals.set_item("failure_type", failure_type).unwrap(); + py.run( + c" +import asyncio +failure = eval(failure_type)('secret manager failed') +cause = RuntimeError('original cause') +context = RuntimeError('original context') +failure.__cause__ = cause +failure.__context__ = context +class Manager: + def sync_read_secret(self, secret_name): + raise failure +manager = Manager() +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let reader = PythonSecretManager::new( + locals.get_item("manager").unwrap().unwrap().unbind(), + None, + None, + ); + (reader, locals.unbind()) + }); + let resolver = SecretResolver::new( + Arc::new(SecretManagerState::new( + SecretManager::External(Arc::new(reader)), + KeyManagementSettings::default(), + )), + Arc::new(move |_: &str| fallback.map(str::to_owned)), + OidcResolver::default(), + ) + .with_failure_policy(FailurePolicy::EnvironmentFallback); + let error = resolver.get_secret("API_KEY", None).await.unwrap_err(); + Python::attach(|py| { + let original = python_error(py, &error).unwrap(); + let locals = locals.bind(py); + assert!( + original + .value(py) + .is(locals.get_item("failure").unwrap().unwrap()) + ); + for (attribute, name) in [("__cause__", "cause"), ("__context__", "context")] { + assert!( + original + .value(py) + .getattr(attribute) + .unwrap() + .is(locals.get_item(name).unwrap().unwrap()) + ); + } + assert!(original.traceback(py).is_some()); + }); + } + + /// Installs a fake `get_secret_from_manager` that records its kwargs, runs `body`, and + /// removes the fake modules again. + fn with_fake_handler<'py>(py: Python<'py>, body: impl FnOnce(&Bound<'py, PyDict>)) { + let locals = PyDict::new(py); + py.run( + c" +import sys, types +calls = [] +def get_secret_from_manager(**kwargs): + calls.append(kwargs) + return 'handled-' + kwargs['secret_name'] +handler = types.ModuleType('litellm.secret_managers.secret_manager_handler') +handler.get_secret_from_manager = get_secret_from_manager +installed = {} +for name in ('litellm', 'litellm.secret_managers'): + if name not in sys.modules: + sys.modules[name] = types.ModuleType(name) + installed[name] = True +sys.modules['litellm.secret_managers.secret_manager_handler'] = handler +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + body(&locals); + py.run( + c" +sys.modules.pop('litellm.secret_managers.secret_manager_handler', None) +for name in installed: + sys.modules.pop(name, None) +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + } + + #[rstest] + #[case::google_kms(KeyManagementSystem::GoogleKms)] + #[case::azure_key_vault(KeyManagementSystem::AzureKeyVault)] + #[case::aws_secret_manager(KeyManagementSystem::AwsSecretManager)] + #[case::google_secret_manager(KeyManagementSystem::GoogleSecretManager)] + #[case::hashicorp_vault(KeyManagementSystem::HashicorpVault)] + #[case::cyberark(KeyManagementSystem::Cyberark)] + #[case::local(KeyManagementSystem::Local)] + #[case::aws_kms(KeyManagementSystem::AwsKms)] + #[case::custom(KeyManagementSystem::Custom)] + fn python_names_round_trip_through_serde(#[case] system: KeyManagementSystem) { + assert_eq!( + serde_json::to_value(system).unwrap(), + serde_json::Value::String(python_name(system).to_owned()) + ); + } + + #[rstest] + #[case::legacy(None, false)] + #[case::custom(Some(KeyManagementSystem::Custom), true)] + fn direct_readers_receive_compatible_kwargs( + #[case] system: Option, + #[case] expects_optional_params: bool, + ) { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c" +class Settings: + def model_dump(self): + return {'scope': 'custom'} +class Manager: + def __init__(self): + self.names = [] + self.optional_params = [] + def sync_read_secret(self, secret_name, optional_params=None, timeout=None): + self.names.append(secret_name) + self.optional_params.append(optional_params) + return 'direct-' + secret_name +manager = Manager() +settings = Settings() +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let manager = locals.get_item("manager").unwrap().unwrap(); + let settings = expects_optional_params + .then(|| locals.get_item("settings").unwrap().unwrap().unbind()); + let reader = PythonSecretManager::new(manager.clone().unbind(), system, settings); + assert_eq!( + reader.read(py, "API_KEY").unwrap().as_deref(), + Some("direct-API_KEY") + ); + assert_eq!( + manager + .getattr("names") + .unwrap() + .extract::>() + .unwrap(), + ["API_KEY"] + ); + let optional_params = manager + .getattr("optional_params") + .unwrap() + .get_item(0) + .unwrap(); + if expects_optional_params { + assert_eq!( + optional_params + .get_item("scope") + .unwrap() + .extract::() + .unwrap(), + "custom" + ); + } else { + assert!(optional_params.is_none()); + } + }); + } + + #[test] + fn configured_systems_dispatch_through_the_python_handler_with_the_original_settings() { + Python::initialize(); + Python::attach(|py| { + with_fake_handler(py, |locals| { + let client = py.eval(c"object()", None, None).unwrap(); + let settings = py.eval(c"object()", None, None).unwrap(); + let reader = PythonSecretManager::new( + client.clone().unbind(), + Some(KeyManagementSystem::AzureKeyVault), + Some(settings.clone().unbind()), + ); + assert_eq!( + reader.read(py, "API_KEY").unwrap().as_deref(), + Some("handled-API_KEY") + ); + assert!(py.import(HANDLER_MODULE).is_ok()); + let calls = locals.get_item("calls").unwrap().unwrap(); + let call = calls.get_item(0).unwrap().cast_into::().unwrap(); + assert!(call.get_item("client").unwrap().unwrap().is(&client)); + assert!( + call.get_item("key_management_settings") + .unwrap() + .unwrap() + .is(&settings) + ); + assert_eq!( + call.get_item("key_manager") + .unwrap() + .unwrap() + .extract::() + .unwrap(), + "azure_key_vault" + ); + assert_eq!( + call.get_item("secret_name") + .unwrap() + .unwrap() + .extract::() + .unwrap(), + "API_KEY" + ); + }); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/secrets/config.rs b/litellm-rust/crates/python-bridge/src/secrets/config.rs new file mode 100644 index 00000000000..6fd380fe40c --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/secrets/config.rs @@ -0,0 +1,338 @@ +use std::sync::Arc; + +use litellm_secrets::{SecretManager, SecretManagerState}; +use litellm_secrets_types::{AccessMode, KeyManagementSettings, KeyManagementSystem, SecretValue}; +use pyo3::prelude::*; +use serde_json::Value; + +use super::callback::PythonSecretManager; +use crate::{ + coercion::{Field, FieldSpec, ProjectionError}, + python_settings::{PythonSettings, Snapshot}, +}; + +const SYSTEM: FieldSpec> = + FieldSpec::new("system", parse_optional_system); +const ACCESS_MODE: FieldSpec = FieldSpec::new("access_mode", parse_access_mode); +const HOSTED_KEYS: FieldSpec>> = + FieldSpec::new("hosted_keys", |field| field.optional_string_collection()); +const STORE_VIRTUAL_KEYS: FieldSpec = + FieldSpec::new("store_virtual_keys", |field| field.truthy()); +const PREFIX_FOR_STORED_VIRTUAL_KEYS: FieldSpec = + FieldSpec::new("prefix_for_stored_virtual_keys", |field| { + field.strict_string() + }); +const PRIMARY_SECRET_NAME: FieldSpec> = + FieldSpec::new("primary_secret_name", |field| field.falsy_optional_string()); +const KMS_KEY_ID: FieldSpec> = + FieldSpec::new("kms_key_id", |field| field.falsy_optional_string()); +const CUSTOM_SECRET_MANAGER: FieldSpec> = + FieldSpec::new("custom_secret_manager", |field| { + field.falsy_optional_string() + }); +const AWS_REGION_NAME: FieldSpec> = + FieldSpec::new("aws_region_name", |field| field.falsy_optional_string()); +const AWS_ROLE_NAME: FieldSpec> = + FieldSpec::new("aws_role_name", |field| field.falsy_optional_string()); +const AWS_SESSION_NAME: FieldSpec> = + FieldSpec::new("aws_session_name", |field| field.falsy_optional_string()); +const AWS_EXTERNAL_ID: FieldSpec> = + FieldSpec::new("aws_external_id", |field| field.falsy_optional_string()); +const AWS_PROFILE_NAME: FieldSpec> = + FieldSpec::new("aws_profile_name", |field| field.falsy_optional_string()); +const AWS_WEB_IDENTITY_TOKEN: FieldSpec> = + FieldSpec::new("aws_web_identity_token", |field| { + field.falsy_optional_string() + }); +const AWS_STS_ENDPOINT: FieldSpec> = + FieldSpec::new("aws_sts_endpoint", |field| field.falsy_optional_string()); +const REPLICA_REGIONS: FieldSpec>> = + FieldSpec::new("replica_regions", |field| { + field.optional_string_collection() + }); +const CLIENT: FieldSpec>> = + FieldSpec::new("client", |field| Ok(field.python_binding())); +const SETTINGS_OBJECT: FieldSpec>> = + FieldSpec::new("settings_object", |field| Ok(field.python_binding())); + +/// `litellm.secret_manager_client` as the bridge classifies it. +#[derive(Debug)] +pub(crate) enum SecretManagerClient { + /// `None`: reads come from the process environment. + Local, + /// A custom manager, legacy compatible client, or manually assigned SDK client that keeps + /// executing in Python. + PythonCallback(Py), +} + +/// One operation-local capture of the secret manager globals, taken while attached to Python. +#[derive(Debug)] +pub(crate) struct SecretManagerSnapshot { + pub(crate) client: SecretManagerClient, + pub(crate) system: Option, + /// Typed settings that drive native routing: access mode and hosted keys. + pub(crate) settings: KeyManagementSettings, + /// The original `KeyManagementSettings` object, handed back to Python callbacks unchanged. + pub(crate) settings_object: Option>, +} + +impl SecretManagerSnapshot { + pub(crate) fn into_state(self) -> Arc { + match self.client { + SecretManagerClient::Local => Arc::new(SecretManagerState::default()), + SecretManagerClient::PythonCallback(client) => Arc::new(SecretManagerState::new( + SecretManager::External(Arc::new(PythonSecretManager::new( + client, + self.system, + self.settings_object, + ))), + self.settings, + )), + } + } +} + +/// Reads and projects the secret manager settings group in one attached operation. +pub(crate) fn read(py: Python<'_>) -> PyResult { + Ok(project(&PythonSettings::SecretManagerBinding.read(py)?)?) +} + +pub(crate) fn project(snapshot: &Snapshot<'_>) -> Result { + let system = snapshot.read(&SYSTEM)?; + let access_mode = snapshot.read(&ACCESS_MODE)?; + let settings = KeyManagementSettings { + hosted_keys: snapshot.read(&HOSTED_KEYS)?, + store_virtual_keys: Some(snapshot.read(&STORE_VIRTUAL_KEYS)?), + prefix_for_stored_virtual_keys: snapshot.read(&PREFIX_FOR_STORED_VIRTUAL_KEYS)?, + access_mode, + primary_secret_name: snapshot.read(&PRIMARY_SECRET_NAME)?, + kms_key_id: snapshot.read(&KMS_KEY_ID)?, + custom_secret_manager: snapshot.read(&CUSTOM_SECRET_MANAGER)?, + aws_region_name: snapshot.read(&AWS_REGION_NAME)?, + aws_role_name: snapshot.read(&AWS_ROLE_NAME)?, + aws_session_name: snapshot.read(&AWS_SESSION_NAME)?, + aws_external_id: snapshot.read(&AWS_EXTERNAL_ID)?.map(SecretValue::new), + aws_profile_name: snapshot.read(&AWS_PROFILE_NAME)?, + aws_web_identity_token: snapshot + .read(&AWS_WEB_IDENTITY_TOKEN)? + .map(SecretValue::new), + aws_sts_endpoint: snapshot.read(&AWS_STS_ENDPOINT)?, + replica_regions: snapshot.read(&REPLICA_REGIONS)?, + ..KeyManagementSettings::default() + }; + let client = match snapshot.read(&CLIENT)? { + None => SecretManagerClient::Local, + Some(client) => SecretManagerClient::PythonCallback(client), + }; + Ok(SecretManagerSnapshot { + client, + system, + settings, + settings_object: snapshot.read(&SETTINGS_OBJECT)?, + }) +} + +fn parse_optional_system( + field: &Field<'_>, +) -> Result, ProjectionError> { + let Some(value) = field.falsy_optional_string()? else { + return Ok(None); + }; + serde_json::from_value(Value::String(value)) + .map(Some) + .map_err(|error| { + ProjectionError::InvalidConfiguration(format!("secret manager system: {error}")) + }) +} + +fn parse_access_mode(field: &Field<'_>) -> Result { + let value = field.strict_string()?; + serde_json::from_value(Value::String(value)).map_err(|error| { + ProjectionError::InvalidConfiguration(format!("secret manager access mode: {error}")) + }) +} + +#[cfg(test)] +mod tests { + use pyo3::{ + prelude::*, + types::{PyDict, PyTuple}, + }; + + use super::{SecretManagerClient, project}; + use crate::python_settings::PythonSettings; + + fn snapshot<'py>( + py: Python<'py>, + system: &str, + access_mode: &str, + store_virtual_keys: Bound<'py, PyAny>, + hosted_keys: Bound<'py, PyAny>, + ) -> crate::python_settings::Snapshot<'py> { + snapshot_with_client( + py, + system, + access_mode, + store_virtual_keys, + hosted_keys, + py.None().into_bound(py), + ) + } + + fn snapshot_with_client<'py>( + py: Python<'py>, + system: &str, + access_mode: &str, + store_virtual_keys: Bound<'py, PyAny>, + hosted_keys: Bound<'py, PyAny>, + client: Bound<'py, PyAny>, + ) -> crate::python_settings::Snapshot<'py> { + let locals = PyDict::new(py); + locals.set_item("client", client).unwrap(); + locals.set_item("system", system).unwrap(); + locals.set_item("access_mode", access_mode).unwrap(); + locals + .set_item("store_virtual_keys", store_virtual_keys) + .unwrap(); + locals.set_item("hosted_keys", hosted_keys).unwrap(); + py.run( + cr#" +from dataclasses import dataclass +from types import SimpleNamespace + +@dataclass(frozen=True, slots=True) +class SecretManager: + system: object + access_mode: object + hosted_keys: object + primary_secret_name: object + store_virtual_keys: object + prefix_for_stored_virtual_keys: object + kms_key_id: object + custom_secret_manager: object + aws_region_name: object + aws_role_name: object + aws_session_name: object + aws_external_id: object + aws_profile_name: object + aws_web_identity_token: object + aws_sts_endpoint: object + replica_regions: object + client: object + settings_object: object + +root = SimpleNamespace(secret_manager=SecretManager( + system=system, + access_mode=access_mode, + hosted_keys=hosted_keys, + primary_secret_name=None, + store_virtual_keys=store_virtual_keys, + prefix_for_stored_virtual_keys="litellm/", + kms_key_id=None, + custom_secret_manager=None, + aws_region_name=None, + aws_role_name=None, + aws_session_name=None, + aws_external_id=None, + aws_profile_name=None, + aws_web_identity_token=None, + aws_sts_endpoint=None, + replica_regions=None, + client=client, + settings_object=None, +)) +"#, + Some(&locals), + Some(&locals), + ) + .unwrap(); + PythonSettings::SecretManagerBinding.snapshot( + locals + .get_item("root") + .unwrap() + .unwrap() + .getattr("secret_manager") + .unwrap(), + ) + } + + #[rstest::rstest] + #[case::string_true(Some("true"), false, true)] + #[case::string_one(Some("1"), false, true)] + #[case::true_value(None, true, true)] + #[case::false_value(None, false, false)] + #[case::string_false(Some("false"), false, true)] + fn python_compatible_boolean_coercion( + #[case] string_value: Option<&str>, + #[case] bool_value: bool, + #[case] expected: bool, + ) { + Python::initialize(); + Python::attach(|py| { + let store_virtual_keys = match string_value { + Some(value) => value.into_pyobject(py).unwrap().into_any(), + None => bool_value.into_pyobject(py).unwrap().to_owned().into_any(), + }; + let hosted_keys = PyTuple::new(py, ["ONE"]).unwrap().into_any(); + let projected = project(&snapshot( + py, + "local", + "read_only", + store_virtual_keys, + hosted_keys, + )) + .unwrap(); + assert_eq!(projected.settings.store_virtual_keys, Some(expected)); + }); + } + + #[test] + fn unknown_system_is_rejected() { + Python::initialize(); + Python::attach(|py| { + let error = project(&snapshot( + py, + "unknown", + "read_only", + false.into_pyobject(py).unwrap().to_owned().into_any(), + PyTuple::empty(py).into_any(), + )) + .unwrap_err(); + let error: PyErr = error.into(); + assert!(error.is_instance_of::(py)); + }); + } + + #[test] + fn client_identity_selects_local_or_python_callback() { + Python::initialize(); + Python::attach(|py| { + let falsy = false.into_pyobject(py).unwrap().to_owned().into_any(); + let local = project(&snapshot( + py, + "local", + "read_only", + falsy.clone(), + PyTuple::empty(py).into_any(), + )) + .unwrap(); + assert!(matches!(local.client, SecretManagerClient::Local)); + assert!(local.settings_object.is_none()); + + let manager = py.eval(c"object()", None, None).unwrap(); + let custom = project(&snapshot_with_client( + py, + "custom", + "read_only", + falsy, + PyTuple::empty(py).into_any(), + manager.clone(), + )) + .unwrap(); + let SecretManagerClient::PythonCallback(client) = custom.client else { + panic!("a live client must stay a Python callback"); + }; + assert!(client.bind(py).is(&manager)); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/secrets/error.rs b/litellm-rust/crates/python-bridge/src/secrets/error.rs new file mode 100644 index 00000000000..1bf9351ee64 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/secrets/error.rs @@ -0,0 +1,27 @@ +use std::fmt; + +use litellm_secrets::Error; +use pyo3::{exceptions::PyBaseException, prelude::*}; + +#[derive(thiserror::Error)] +#[error("Python secret manager failed")] +struct PythonSecretError(Py); + +impl fmt::Debug for PythonSecretError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("PythonSecretError") + } +} + +pub(super) fn external_error(py: Python<'_>, error: PyErr) -> Error { + Error::ExternalManager(Box::new(PythonSecretError(error.into_value(py)))) +} + +pub(crate) fn python_error(py: Python<'_>, error: &Error) -> Option { + let Error::ExternalManager(source) = error else { + return None; + }; + source + .downcast_ref::() + .map(|error| PyErr::from_value(error.0.clone_ref(py).into_bound(py).into_any())) +} diff --git a/litellm-rust/crates/python-bridge/src/secrets/mod.rs b/litellm-rust/crates/python-bridge/src/secrets/mod.rs new file mode 100644 index 00000000000..c753015aa9e --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/secrets/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod callback; +pub(crate) mod config; +mod error; +pub(crate) mod resolved; + +pub(crate) use error::python_error; diff --git a/litellm-rust/crates/python-bridge/src/secrets/resolved.rs b/litellm-rust/crates/python-bridge/src/secrets/resolved.rs new file mode 100644 index 00000000000..877c429169a --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/secrets/resolved.rs @@ -0,0 +1,252 @@ +use std::{collections::HashMap, sync::Arc}; + +use futures_util::{future::BoxFuture, future::try_join_all}; +use litellm_core_utils::settings::{Lookup, ProcessEnvironment}; +use litellm_llms::base_llm::inference::secrets::{SecretSource, Secrets}; +use litellm_secrets::{ + Error, FailurePolicy, OidcResolver, Secret, SecretManagerState, SecretResolver, +}; + +use super::config::SecretManagerSnapshot; + +pub(crate) struct ResolvedSecrets { + resolver: SecretResolver, +} + +impl ResolvedSecrets { + pub(crate) fn new(snapshot: SecretManagerSnapshot) -> Self { + Self::from_state(snapshot.into_state()) + } + + fn from_state(state: Arc) -> Self { + Self { + resolver: SecretResolver::new( + state, + Arc::new(ProcessEnvironment), + OidcResolver::default(), + ) + .with_failure_policy(FailurePolicy::EnvironmentFallback), + } + } +} + +impl SecretSource for ResolvedSecrets { + fn resolve<'a>(&'a self, names: &'a [&'static str]) -> BoxFuture<'a, Result> { + Box::pin(async move { + let values = try_join_all(names.iter().map(|name| async move { + self.resolver + .get_secret(name, None) + .await + .map(|secret| secret.map(|secret| ((*name).to_owned(), secret_value(secret)))) + })) + .await? + .into_iter() + .flatten() + .collect::>(); + Ok(Arc::new(ResolvedLookup { values }) as Secrets) + }) + } +} + +struct ResolvedLookup { + values: HashMap, +} + +impl Lookup for ResolvedLookup { + fn get(&self, name: &str) -> Option { + self.values + .get(name) + .cloned() + .or_else(|| ProcessEnvironment.get(name)) + } +} + +fn secret_value(secret: Secret) -> String { + match secret { + Secret::String(value) => value.expose().to_owned(), + Secret::Bool(value) => if value { "True" } else { "False" }.to_owned(), + Secret::Json(value) => value.to_string(), + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use aws_sdk_secretsmanager::Client; + use aws_sdk_secretsmanager::config::{ + BehaviorVersion, Credentials, Region, retry::RetryConfig, + }; + use litellm_secrets::{AccessMode, KeyManagementSettings, SecretManager, SecretManagerState}; + use litellm_secrets_aws::AwsSecretsManagerV2; + use serde_json::json; + use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{body_partial_json, header}, + }; + + use super::ResolvedSecrets; + use litellm_llms::base_llm::inference::secrets::SecretSource; + + fn state(server: &MockServer, settings: KeyManagementSettings) -> Arc { + let client = Client::from_conf( + aws_sdk_secretsmanager::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new("us-east-1")) + .credentials_provider(Credentials::new("test", "test", None, None, "test")) + .endpoint_url(server.uri()) + .retry_config(RetryConfig::disabled()) + .build(), + ); + Arc::new(SecretManagerState::new( + SecretManager::AwsSecretsManagerV2(AwsSecretsManagerV2::new( + client, + (&settings).into(), + )), + settings, + )) + } + + async fn resolve(state: Arc, name: &'static str) -> Option { + ResolvedSecrets::from_state(state) + .resolve(&[name]) + .await + .unwrap() + .get(name) + } + + #[tokio::test] + async fn hosted_key_miss_falls_back_to_environment() { + let name = "LITELLM_RUST_BRIDGE_HOSTED_KEY_MISS"; + unsafe { std::env::set_var(name, "env-key") }; + let server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue")) + .and(body_partial_json(json!({"SecretId": name}))) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"SecretString": "manager-key"})), + ) + .expect(0) + .mount(&server) + .await; + let result = resolve( + state( + &server, + KeyManagementSettings { + hosted_keys: Some(vec!["OTHER".into()]), + ..Default::default() + }, + ), + name, + ) + .await; + unsafe { std::env::remove_var(name) }; + assert_eq!(result.as_deref(), Some("env-key")); + assert_eq!(server.received_requests().await.unwrap().len(), 0); + } + + #[tokio::test] + async fn manager_failure_falls_back_to_environment() { + let name = "LITELLM_RUST_BRIDGE_MANAGER_FAILURE"; + unsafe { std::env::set_var(name, "env-key") }; + let server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue")) + .respond_with(ResponseTemplate::new(500)) + .expect(1) + .mount(&server) + .await; + let result = resolve(state(&server, KeyManagementSettings::default()), name).await; + unsafe { std::env::remove_var(name) }; + assert_eq!(result.as_deref(), Some("env-key")); + assert_eq!(server.received_requests().await.unwrap().len(), 1); + + let missing_server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue")) + .respond_with(ResponseTemplate::new(500)) + .expect(1) + .mount(&missing_server) + .await; + let missing = + ResolvedSecrets::from_state(state(&missing_server, KeyManagementSettings::default())) + .resolve(&["LITELLM_RUST_BRIDGE_MANAGER_FAILURE_MISSING"]) + .await; + assert!(matches!(missing, Err(litellm_secrets::Error::Aws(_)))); + } + + #[tokio::test] + async fn write_only_mode_never_consults_the_manager() { + let name = "LITELLM_RUST_BRIDGE_WRITE_ONLY"; + unsafe { std::env::set_var(name, "env-key") }; + let server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue")) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"SecretString": "manager-key"})), + ) + .expect(0) + .mount(&server) + .await; + let result = resolve( + state( + &server, + KeyManagementSettings { + access_mode: AccessMode::WriteOnly, + ..Default::default() + }, + ), + name, + ) + .await; + unsafe { std::env::remove_var(name) }; + assert_eq!(result.as_deref(), Some("env-key")); + assert_eq!(server.received_requests().await.unwrap().len(), 0); + } + + #[tokio::test] + async fn read_only_mode_resolves_from_the_manager() { + let name = "LITELLM_RUST_BRIDGE_READ_ONLY"; + let server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue")) + .and(body_partial_json(json!({"SecretId": name}))) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"SecretString": "manager-key"})), + ) + .expect(1) + .mount(&server) + .await; + assert_eq!( + resolve(state(&server, KeyManagementSettings::default()), name) + .await + .as_deref(), + Some("manager-key") + ); + assert_eq!(server.received_requests().await.unwrap().len(), 1); + } + + #[tokio::test] + async fn oidc_failures_are_not_converted_to_missing_secrets() { + let result = ResolvedSecrets::from_state(Arc::new(SecretManagerState::default())) + .resolve(&["oidc/"]) + .await; + assert!(matches!(result, Err(litellm_secrets::Error::InvalidOidc))); + } + + #[tokio::test] + async fn undeclared_names_still_read_the_process_environment() { + let name = "LITELLM_RUST_BRIDGE_UNDECLARED"; + unsafe { std::env::set_var(name, "env-key") }; + let server = MockServer::start().await; + let result = resolve( + state( + &server, + KeyManagementSettings { + hosted_keys: Some(vec!["OTHER".into()]), + ..Default::default() + }, + ), + name, + ) + .await; + unsafe { std::env::remove_var(name) }; + assert_eq!(result.as_deref(), Some("env-key")); + assert_eq!(server.received_requests().await.unwrap().len(), 0); + } +} diff --git a/litellm-rust/crates/python-bridge/src/token_counter.rs b/litellm-rust/crates/python-bridge/src/token_counter.rs deleted file mode 100644 index 244401e6696..00000000000 --- a/litellm-rust/crates/python-bridge/src/token_counter.rs +++ /dev/null @@ -1,158 +0,0 @@ -use std::sync::Arc; - -#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] -use std::{num::NonZero, thread::available_parallelism}; - -#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] -use litellm_host_python::release_gil; -use litellm_host_python::run_async; -use litellm_token_counter::{ - CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter, -}; -use pyo3::{ - exceptions::{PyRuntimeError, PyValueError}, - prelude::*, - types::PyAny, -}; -use tokio::sync::Semaphore; - -use crate::errors::RustBridgeDeclined; - -/// Counts the input tokens of a raw request body off the Python event loop with -/// the GIL released. Python owns which requests get here and what to do with -/// the count. At most one encode per core runs at a time; the rest wait in the -/// async task, where a cancelled Python awaiter drops them before any blocking -/// work is scheduled. -#[pyclass(frozen)] -pub(crate) struct TokenCounter { - inner: Arc, - encode_slots: Arc, -} - -#[pymethods] -impl TokenCounter { - #[new] - fn new(py: Python<'_>, tokenizer_json: &str) -> PyResult { - #[cfg(feature = "fast")] - { - Self::load(py, || CoreTokenCounter::from_json_fast(tokenizer_json)) - } - #[cfg(all(not(feature = "fast"), feature = "huggingface"))] - { - Self::load(py, || CoreTokenCounter::from_json(tokenizer_json)) - } - #[cfg(not(any(feature = "fast", feature = "huggingface")))] - { - let _ = (py, tokenizer_json); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the fast or huggingface feature", - )) - } - } - - #[staticmethod] - fn from_cl100k_ranks(py: Python<'_>, rank_file: &str) -> PyResult { - #[cfg(feature = "fast")] - { - Self::load(py, || CoreTokenCounter::from_cl100k_ranks(rank_file)) - } - #[cfg(not(feature = "fast"))] - { - let _ = (py, rank_file); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the fast feature", - )) - } - } - - #[staticmethod] - fn from_o200k_ranks(py: Python<'_>, rank_file: &str) -> PyResult { - #[cfg(feature = "fast")] - { - Self::load(py, || CoreTokenCounter::from_o200k_ranks(rank_file)) - } - #[cfg(not(feature = "fast"))] - { - let _ = (py, rank_file); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the fast feature", - )) - } - } - - #[staticmethod] - fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult { - #[cfg(feature = "tiktoken")] - { - Self::load(py, || CoreTokenCounter::from_tiktoken(encoding)) - } - #[cfg(not(feature = "tiktoken"))] - { - let _ = (py, encoding); - Err(RustBridgeDeclined::new_err( - "tokenizer backend requires the tiktoken feature", - )) - } - } - - fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult> { - let counter = Arc::clone(&self.inner); - let encode_slots = Arc::clone(&self.encode_slots); - let body = body.to_vec(); - run_async( - py, - async move { - let _slot = encode_slots - .acquire_owned() - .await - .map_err(|error| Error::Task(error.to_string()))?; - tokio::task::spawn_blocking(move || count_body(&counter, &body)) - .await - .map_err(|error| Error::Task(error.to_string()))? - }, - token_count_error_to_pyerr, - ) - } -} - -impl TokenCounter { - #[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] - fn load( - py: Python<'_>, - load: impl FnOnce() -> Result + Send, - ) -> PyResult { - let inner = release_gil(py, load).map_err(token_count_error_to_pyerr)?; - Ok(Self { - inner: Arc::new(inner), - encode_slots: Arc::new(Semaphore::new(encode_parallelism())), - }) - } -} - -#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] -fn encode_parallelism() -> usize { - available_parallelism().map_or(1, NonZero::get) -} - -fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result { - let request = CountableRequest::parse(body)?; - counter.count_request(&request) -} - -fn token_count_error_to_pyerr(error: Error) -> PyErr { - let message = error.to_string(); - match error { - Error::Load(_) - | Error::Ranks(_) - | Error::UnicodeClasses - | Error::UnsupportedTokenizer(_) => PyValueError::new_err(message), - Error::RequestParse(_) - | Error::MissingInput - | Error::FloatText - | Error::ContentBlock - | Error::ArrayItems - | Error::JsonSerialization(_) - | Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message), - Error::Encode(_) | Error::Task(_) => PyRuntimeError::new_err(message), - } -} diff --git a/litellm-rust/crates/python-bridge/src/tokenizer.rs b/litellm-rust/crates/python-bridge/src/tokenizer.rs new file mode 100644 index 00000000000..df219d55eb6 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/tokenizer.rs @@ -0,0 +1,713 @@ +//! The Python face of the text codecs: one `Tokenizer` class over the tiktoken and Hugging +//! Face backends, carrying the read-only surface of `tiktoken.Encoding` and +//! `tokenizers.Tokenizer` that `litellm/litellm_core_utils/tokenizer.py` wraps. +use std::borrow::Cow; +#[cfg(any(feature = "tiktoken", feature = "huggingface"))] +use std::collections::HashMap; +use std::sync::Arc; +#[cfg(feature = "fast")] +use std::sync::OnceLock; + +use litellm_host_python::{enter_native, release_gil}; +#[cfg(feature = "fast")] +use litellm_token_counter::fast::{FastCounter, FastTokenizer}; +use litellm_token_counter::{Error, TextCodec}; +use pyo3::{exceptions::PyUnicodeEncodeError, prelude::*, types::PyString}; + +#[cfg(any(feature = "tiktoken", feature = "huggingface"))] +use pyo3::exceptions::PyValueError; +#[cfg(feature = "huggingface")] +use pyo3::{exceptions::PyIOError, types::PyDict}; +#[cfg(feature = "tiktoken")] +use pyo3::{ + exceptions::{PyKeyError, PyRuntimeError}, + types::PyBytes, +}; + +#[cfg(not(all(feature = "tiktoken", feature = "huggingface")))] +use crate::errors::RustBridgeDeclined; +use crate::routes::token_counter::token_count_error_to_pyerr; + +#[cfg(feature = "huggingface")] +use litellm_token_counter::huggingface::{ + EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection, PaddingStrategy, + TruncationDirection, encoding_from_json, encoding_to_json, +}; +#[cfg(feature = "tiktoken")] +use litellm_token_counter::tiktoken::{TiktokenTokenizer, Vocabulary}; + +#[cfg(feature = "tiktoken")] +pub(crate) fn load_tiktoken(py: Python<'_>, encoding: &str) -> PyResult { + enter_native()?; + let resource: std::path::PathBuf = + PyModule::import(py, "litellm.litellm_core_utils.tokenizers")? + .getattr("__file__")? + .extract()?; + release_gil(py, || { + TiktokenTokenizer::from_cached_ranks(encoding, |file| { + std::fs::read_to_string(resource.with_file_name(file)) + }) + }) + .map_err(|error| token_count_error_to_pyerr(error.into())) +} + +pub(crate) enum Codec { + #[cfg(feature = "tiktoken")] + Tiktoken(TiktokenTokenizer), + #[cfg(feature = "huggingface")] + HuggingFace(HuggingFaceTokenizer), +} + +impl Codec { + pub(crate) fn codec(&self) -> &dyn TextCodec { + match *self { + #[cfg(feature = "tiktoken")] + Self::Tiktoken(ref tokenizer) => tokenizer, + #[cfg(feature = "huggingface")] + Self::HuggingFace(ref tokenizer) => tokenizer, + } + } + + #[cfg(feature = "fast")] + fn fast_counter(&self) -> Option { + match *self { + #[cfg(feature = "tiktoken")] + Self::Tiktoken(ref tokenizer) => tokenizer.fast_counter(), + #[cfg(feature = "huggingface")] + Self::HuggingFace(ref tokenizer) => tokenizer.fast_counter(), + } + } +} + +/// The loaded model is shared: `TokenCounter::from_tokenizer` counts with the same parse, +/// and the opt-in count-only counter is derived from it once, on first use. +#[pyclass(frozen, module = "litellm.rust_bridge._native")] +pub(crate) struct Tokenizer { + inner: Arc, + #[cfg(feature = "fast")] + fast: OnceLock>>, +} + +#[pymethods] +impl Tokenizer { + #[staticmethod] + fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult { + #[cfg(feature = "tiktoken")] + { + let tokenizer = load_tiktoken(py, encoding)?; + Ok(Self::new(Codec::Tiktoken(tokenizer))) + } + #[cfg(not(feature = "tiktoken"))] + { + let _ = (py, encoding); + Err(RustBridgeDeclined::new_err( + "tokenizer backend requires the tiktoken feature", + )) + } + } + + #[staticmethod] + fn from_json(py: Python<'_>, tokenizer_json: &str) -> PyResult { + #[cfg(feature = "huggingface")] + { + enter_native()?; + let tokenizer = release_gil(py, || HuggingFaceTokenizer::from_json(tokenizer_json)) + .map_err(|error| token_count_error_to_pyerr(error.into()))?; + Ok(Self::new(Codec::HuggingFace(tokenizer))) + } + #[cfg(not(feature = "huggingface"))] + { + let _ = (py, tokenizer_json); + Err(RustBridgeDeclined::new_err( + "tokenizer backend requires the huggingface feature", + )) + } + } + + #[staticmethod] + #[pyo3(signature = (identifier, revision = "main", token = None))] + fn from_pretrained( + py: Python<'_>, + identifier: &str, + revision: &str, + token: Option<&str>, + ) -> PyResult { + #[cfg(feature = "huggingface")] + { + enter_native()?; + let kwargs = PyDict::new(py); + kwargs.set_item("repo_id", identifier)?; + kwargs.set_item("filename", "tokenizer.json")?; + kwargs.set_item("revision", revision)?; + kwargs.set_item("token", token)?; + let path: String = PyModule::import(py, "huggingface_hub")? + .getattr("hf_hub_download")? + .call((), Some(&kwargs))? + .extract()?; + let json = + release_gil(py, || std::fs::read_to_string(path)).map_err(PyIOError::new_err)?; + Self::from_json(py, &json) + } + #[cfg(not(feature = "huggingface"))] + { + let _ = (py, identifier, revision, token); + Err(RustBridgeDeclined::new_err( + "tokenizer backend requires the huggingface feature", + )) + } + } + + fn encode(&self, py: Python<'_>, text: &Bound<'_, PyString>) -> PyResult> { + enter_native()?; + let text = self.text(text)?; + release_gil(py, || self.inner.codec().encode(&text)).map_err(token_count_error_to_pyerr) + } + + #[pyo3(signature = (ids, skip_special_tokens = true))] + fn decode(&self, py: Python<'_>, ids: Vec, skip_special_tokens: bool) -> PyResult { + enter_native()?; + release_gil(py, || self.inner.codec().decode(&ids, skip_special_tokens)) + .map_err(token_count_error_to_pyerr) + } + + #[pyo3(signature = (text, fast = false))] + fn count(&self, py: Python<'_>, text: &Bound<'_, PyString>, fast: bool) -> PyResult { + enter_native()?; + let text = self.text(text)?; + let counter = self.counter(py, fast); + release_gil(py, || { + litellm_token_counter::Tokenizer::count_tokens(&counter, &text) + }) + .map_err(token_count_error_to_pyerr) + } + + #[getter] + fn name(&self) -> &str { + self.inner.codec().name() + } + + // ---- tiktoken: the `tiktoken.Encoding` surface ------------------------------------------ + + #[cfg(feature = "tiktoken")] + fn encode_special( + &self, + py: Python<'_>, + text: &Bound<'_, PyString>, + allowed: Vec, + ) -> PyResult> { + enter_native()?; + let tokenizer = self.tiktoken()?; + let text = self.text(text)?; + release_gil(py, || tokenizer.encode_special(&text, &allowed)) + .map_err(PyRuntimeError::new_err) + } + + /// tiktoken's `encode_with_unstable`: `(stable_tokens, completions)`. + #[cfg(feature = "tiktoken")] + fn encode_with_unstable( + &self, + py: Python<'_>, + text: &Bound<'_, PyString>, + allowed: Vec, + ) -> PyResult<(Vec, Vec>)> { + enter_native()?; + let tokenizer = self.tiktoken()?; + let text = self.text(text)?; + Ok(release_gil(py, || { + tokenizer.encode_with_unstable(&text, &allowed) + })) + } + + /// The special tokens by text: tiktoken's `_special_tokens`. + #[cfg(feature = "tiktoken")] + fn special_tokens(&self) -> PyResult> { + Ok(self + .vocabulary()? + .special_tokens() + .map(|(token, rank)| (token.to_owned(), rank)) + .collect()) + } + + #[cfg(feature = "tiktoken")] + fn max_token_value(&self) -> PyResult { + Ok(self.vocabulary()?.max_token_value()) + } + + #[cfg(feature = "tiktoken")] + fn is_special_token(&self, token: u32) -> PyResult { + Ok(self.vocabulary()?.is_special_token(token)) + } + + /// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`. + #[cfg(feature = "tiktoken")] + fn token_byte_values<'py>(&self, py: Python<'py>) -> PyResult>> { + let vocabulary = self.vocabulary()?; + let values = release_gil(py, || vocabulary.token_byte_values()); + Ok(values.iter().map(|value| PyBytes::new(py, value)).collect()) + } + + /// The token of one whole piece; `KeyError` when it is not in the vocabulary. + #[cfg(feature = "tiktoken")] + fn encode_single_token(&self, py: Python<'_>, piece: Vec) -> PyResult { + self.vocabulary()? + .encode_single_token(&piece) + .ok_or_else(|| PyKeyError::new_err(PyBytes::new(py, &piece).unbind())) + } + + #[cfg(feature = "tiktoken")] + fn decode_bytes<'py>(&self, py: Python<'py>, ids: Vec) -> PyResult> { + enter_native()?; + let tokenizer = self.tiktoken()?; + let bytes = + release_gil(py, || tokenizer.decode_bytes(&ids)).map_err(PyKeyError::new_err)?; + Ok(PyBytes::new(py, &bytes)) + } + + // ---- Hugging Face: the `tokenizers.Tokenizer` surface ----------------------------------- + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (sequence, pair = None, is_pretokenized = false, add_special_tokens = true, fast = false))] + fn encode_huggingface( + &self, + py: Python<'_>, + sequence: Sequence, + pair: Option, + is_pretokenized: bool, + add_special_tokens: bool, + fast: bool, + ) -> PyResult { + enter_native()?; + let tokenizer = self.huggingface()?; + let sequence = sequence.input(is_pretokenized)?; + let input = match pair { + Some(pair) => EncodeInput::Dual(sequence, pair.input(is_pretokenized)?), + None => EncodeInput::Single(sequence), + }; + release_gil(py, || { + tokenizer.encode_result(input, add_special_tokens, fast) + }) + .map(|inner| HuggingFaceEncoding { inner }) + .map_err(|error| token_count_error_to_pyerr(Error::from(error))) + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (inputs, is_pretokenized = false, add_special_tokens = true, fast = false))] + fn encode_batch_huggingface( + &self, + py: Python<'_>, + inputs: Vec<(Sequence, Option)>, + is_pretokenized: bool, + add_special_tokens: bool, + fast: bool, + ) -> PyResult> { + enter_native()?; + let tokenizer = self.huggingface()?; + let inputs = inputs + .into_iter() + .map(|(sequence, pair)| { + let sequence = sequence.input(is_pretokenized)?; + match pair { + Some(pair) => Ok(EncodeInput::Dual(sequence, pair.input(is_pretokenized)?)), + None => Ok(EncodeInput::Single(sequence)), + } + }) + .collect::>>()?; + release_gil(py, || { + tokenizer.encode_batch_result(inputs, add_special_tokens, fast) + }) + .map(|encodings| { + encodings + .into_iter() + .map(|inner| HuggingFaceEncoding { inner }) + .collect() + }) + .map_err(|error| token_count_error_to_pyerr(Error::from(error))) + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (pretty = false))] + fn to_json(&self, py: Python<'_>, pretty: bool) -> PyResult { + enter_native()?; + let tokenizer = self.huggingface()?; + release_gil(py, || tokenizer.to_json(pretty)) + .map_err(|error| token_count_error_to_pyerr(Error::from(error))) + } + + #[cfg(feature = "huggingface")] + fn token_to_id(&self, token: &str) -> PyResult> { + Ok(self.huggingface()?.token_to_id(token)) + } + + #[cfg(feature = "huggingface")] + fn id_to_token(&self, id: u32) -> PyResult> { + Ok(self.huggingface()?.id_to_token(id)) + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (with_added_tokens = true))] + fn get_vocab(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult> { + let tokenizer = self.huggingface()?; + Ok(release_gil(py, || tokenizer.vocab(with_added_tokens))) + } + + #[cfg(feature = "huggingface")] + #[pyo3(signature = (with_added_tokens = true))] + fn get_vocab_size(&self, with_added_tokens: bool) -> PyResult { + Ok(self.huggingface()?.vocab_size(with_added_tokens)) + } + + /// The added tokens by id as `(id, (content, single_word, lstrip, rstrip, normalized, + /// special))`, for Python to rebuild as `tokenizers.AddedToken`. + #[cfg(feature = "huggingface")] + fn added_tokens_decoder(&self) -> PyResult> { + Ok(self + .huggingface()? + .added_tokens_decoder() + .into_iter() + .map(|(id, token)| { + ( + id, + ( + token.content, + token.single_word, + token.lstrip, + token.rstrip, + token.normalized, + token.special, + ), + ) + }) + .collect()) + } + + /// The padding parameters as `tokenizers.Tokenizer.padding` reports them. + #[cfg(feature = "huggingface")] + fn padding<'py>(&self, py: Python<'py>) -> PyResult>> { + let Some(params) = self.huggingface()?.padding() else { + return Ok(None); + }; + let padding = PyDict::new(py); + padding.set_item( + "length", + match params.strategy { + PaddingStrategy::BatchLongest => None, + PaddingStrategy::Fixed(length) => Some(length), + }, + )?; + padding.set_item("pad_to_multiple_of", params.pad_to_multiple_of)?; + padding.set_item("pad_id", params.pad_id)?; + padding.set_item("pad_type_id", params.pad_type_id)?; + padding.set_item("pad_token", ¶ms.pad_token)?; + padding.set_item("direction", params.direction.as_ref())?; + Ok(Some(padding)) + } + + /// The truncation parameters as `tokenizers.Tokenizer.truncation` reports them. + #[cfg(feature = "huggingface")] + fn truncation<'py>(&self, py: Python<'py>) -> PyResult>> { + let Some(params) = self.huggingface()?.truncation() else { + return Ok(None); + }; + let truncation = PyDict::new(py); + truncation.set_item("max_length", params.max_length)?; + truncation.set_item("stride", params.stride)?; + truncation.set_item("strategy", params.strategy.as_ref())?; + truncation.set_item("direction", params.direction.as_ref())?; + Ok(Some(truncation)) + } + + #[cfg(feature = "huggingface")] + fn num_special_tokens_to_add(&self, is_pair: bool) -> PyResult { + Ok(self.huggingface()?.num_special_tokens_to_add(is_pair)) + } + + #[cfg(feature = "huggingface")] + fn encode_special_tokens(&self) -> PyResult { + Ok(self.huggingface()?.encode_special_tokens()) + } +} + +#[cfg(feature = "huggingface")] +type AddedTokenFields = (String, bool, bool, bool, bool, bool); + +impl Tokenizer { + fn new(inner: Codec) -> Self { + Self { + inner: Arc::new(inner), + #[cfg(feature = "fast")] + fast: OnceLock::new(), + } + } + + pub(crate) fn counter(&self, py: Python<'_>, fast: bool) -> SharedCounter { + #[cfg(feature = "fast")] + if fast { + let counter = self.fast.get().unwrap_or_else(|| { + release_gil(py, || { + self.fast + .get_or_init(|| self.inner.fast_counter().map(Arc::new)) + }) + }); + if let Some(counter) = counter { + return SharedCounter::Fast(Arc::clone(counter)); + } + } + #[cfg(not(feature = "fast"))] + let _ = (py, fast); + SharedCounter::Codec(Arc::clone(&self.inner)) + } + + /// A Python `str` as UTF-8. tiktoken replaces lone surrogates the way its Python `encode` + /// does; `tokenizers` rejects them, so that backend keeps the encode error. + fn text<'a>(&self, text: &'a Bound<'_, PyString>) -> PyResult> { + match text.to_cow() { + Ok(text) => Ok(text), + Err(error) => match *self.inner { + #[cfg(feature = "tiktoken")] + Codec::Tiktoken(_) if error.is_instance_of::(text.py()) => { + text.call_method1("encode", ("utf-16", "surrogatepass"))? + .call_method1("decode", ("utf-16", "replace"))? + .extract::() + .map(Cow::Owned) + } + _ => Err(error), + }, + } + } + + #[cfg(feature = "tiktoken")] + fn tiktoken(&self) -> PyResult<&TiktokenTokenizer> { + match *self.inner { + Codec::Tiktoken(ref tokenizer) => Ok(tokenizer), + #[cfg(feature = "huggingface")] + Codec::HuggingFace(_) => Err(PyValueError::new_err("requires a tiktoken encoding")), + } + } + + #[cfg(feature = "tiktoken")] + fn vocabulary(&self) -> PyResult<&Vocabulary> { + self.tiktoken()?.vocabulary().ok_or_else(|| { + PyRuntimeError::new_err("this encoding was built without its vocabulary") + }) + } + + #[cfg(feature = "huggingface")] + fn huggingface(&self) -> PyResult<&HuggingFaceTokenizer> { + match *self.inner { + Codec::HuggingFace(ref tokenizer) => Ok(tokenizer), + #[cfg(feature = "tiktoken")] + Codec::Tiktoken(_) => Err(PyValueError::new_err("requires a Hugging Face tokenizer")), + } + } +} + +pub(crate) enum SharedCounter { + Codec(Arc), + #[cfg(feature = "fast")] + Fast(Arc), +} + +impl litellm_token_counter::Tokenizer for SharedCounter { + fn count_tokens(&self, text: &str) -> Result { + match self { + Self::Codec(codec) => codec.codec().count_tokens(text), + #[cfg(feature = "fast")] + Self::Fast(counter) => counter.count_tokens(text).map_err(Error::from), + } + } +} + +#[cfg(feature = "huggingface")] +#[derive(FromPyObject)] +pub(crate) enum Sequence { + Text(String), + Words(Vec), +} + +#[cfg(feature = "huggingface")] +impl Sequence { + fn input(self, is_pretokenized: bool) -> PyResult> { + match (self, is_pretokenized) { + (Self::Text(text), false) => Ok(text.into()), + (Self::Words(words), true) => Ok(words.into()), + _ => Err(pyo3::exceptions::PyTypeError::new_err( + "input must match is_pretokenized", + )), + } + } +} + +#[cfg(feature = "huggingface")] +fn direction(value: &str, left: T, right: T, what: &str) -> PyResult { + match value { + "left" => Ok(left), + "right" => Ok(right), + other => Err(PyValueError::new_err(format!( + "invalid {what} direction {other:?}: expected 'left' or 'right'" + ))), + } +} + +/// `tokenizers.Encoding`, mutable like the original: `pad`, `truncate` and `set_sequence_id` +/// change it in place. +#[cfg(feature = "huggingface")] +#[pyclass(module = "litellm.rust_bridge._native")] +pub(crate) struct HuggingFaceEncoding { + inner: Encoding, +} + +#[cfg(feature = "huggingface")] +#[pymethods] +impl HuggingFaceEncoding { + #[new] + #[pyo3(signature = (json = None))] + fn new(json: Option<&str>) -> PyResult { + let inner = match json { + Some(json) => encoding_from_json(json) + .map_err(|error| PyValueError::new_err(error.to_string()))?, + None => Encoding::default(), + }; + Ok(Self { inner }) + } + + #[staticmethod] + #[pyo3(signature = (encodings, growing_offsets = true))] + fn merge(encodings: Vec>, growing_offsets: bool) -> Self { + Self { + inner: Encoding::merge( + encodings.iter().map(|encoding| encoding.inner.clone()), + growing_offsets, + ), + } + } + + fn __reduce__<'py>( + &self, + py: Python<'py>, + ) -> PyResult<(Bound<'py, pyo3::types::PyType>, (String,))> { + let json = encoding_to_json(&self.inner) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + Ok((py.get_type::(), (json,))) + } + + fn __repr__(&self) -> String { + format!( + "Encoding(num_tokens={}, attributes=[ids, type_ids, tokens, offsets, \ + attention_mask, special_tokens_mask, overflowing])", + self.inner.len() + ) + } + + fn __len__(&self) -> usize { + self.inner.len() + } + #[getter] + fn ids(&self) -> Vec { + self.inner.get_ids().to_vec() + } + #[getter] + fn tokens(&self) -> Vec { + self.inner.get_tokens().to_vec() + } + #[getter] + fn offsets(&self) -> Vec<(usize, usize)> { + self.inner.get_offsets().to_vec() + } + #[getter] + fn type_ids(&self) -> Vec { + self.inner.get_type_ids().to_vec() + } + #[getter] + fn attention_mask(&self) -> Vec { + self.inner.get_attention_mask().to_vec() + } + #[getter] + fn special_tokens_mask(&self) -> Vec { + self.inner.get_special_tokens_mask().to_vec() + } + #[getter] + fn word_ids(&self) -> Vec> { + self.inner.get_word_ids().to_vec() + } + #[getter] + fn sequence_ids(&self) -> Vec> { + self.inner.get_sequence_ids() + } + #[getter] + fn overflowing(&self) -> Vec { + self.inner + .get_overflowing() + .iter() + .cloned() + .map(|inner| Self { inner }) + .collect() + } + #[getter] + fn n_sequences(&self) -> usize { + self.inner.n_sequences() + } + + #[pyo3(signature = (word_index, sequence_index = 0))] + fn word_to_tokens(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> { + self.inner.word_to_tokens(word_index, sequence_index) + } + #[pyo3(signature = (word_index, sequence_index = 0))] + fn word_to_chars(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> { + self.inner.word_to_chars(word_index, sequence_index) + } + fn token_to_sequence(&self, token_index: usize) -> Option { + self.inner.token_to_sequence(token_index) + } + fn token_to_chars(&self, token_index: usize) -> Option<(usize, usize)> { + self.inner + .token_to_chars(token_index) + .map(|(_, offsets)| offsets) + } + fn token_to_word(&self, token_index: usize) -> Option { + self.inner.token_to_word(token_index).map(|(_, word)| word) + } + #[pyo3(signature = (char_pos, sequence_index = 0))] + fn char_to_token(&self, char_pos: usize, sequence_index: usize) -> Option { + self.inner.char_to_token(char_pos, sequence_index) + } + #[pyo3(signature = (char_pos, sequence_index = 0))] + fn char_to_word(&self, char_pos: usize, sequence_index: usize) -> Option { + self.inner.char_to_word(char_pos, sequence_index) + } + + fn set_sequence_id(&mut self, sequence_id: usize) { + self.inner.set_sequence_id(sequence_id); + } + + #[pyo3(signature = (length, direction = "right", pad_id = 0, pad_type_id = 0, pad_token = "[PAD]"))] + fn pad( + &mut self, + length: usize, + direction: &str, + pad_id: u32, + pad_type_id: u32, + pad_token: &str, + ) -> PyResult<()> { + let direction = self::direction( + direction, + PaddingDirection::Left, + PaddingDirection::Right, + "padding", + )?; + self.inner + .pad(length, pad_id, pad_type_id, pad_token, direction); + Ok(()) + } + + #[pyo3(signature = (max_length, stride = 0, direction = "right"))] + fn truncate(&mut self, max_length: usize, stride: usize, direction: &str) -> PyResult<()> { + let direction = self::direction( + direction, + TruncationDirection::Left, + TruncationDirection::Right, + "truncation", + )?; + self.inner.truncate(max_length, stride, direction); + Ok(()) + } +} diff --git a/litellm-rust/crates/python-compat/AGENTS.md b/litellm-rust/crates/python-compat/AGENTS.md new file mode 100644 index 00000000000..0869568d33b --- /dev/null +++ b/litellm-rust/crates/python-compat/AGENTS.md @@ -0,0 +1,23 @@ +- Pure Python *data formats* in Rust, for state Python LiteLLM writes and Rust must read or write byte-compatibly + - No PyO3, no live objects: truthiness, `__str__`, descriptors of real Python objects belong to `python-bridge`'s coercion layer + - Format *choices* stay with callers: the `{timestamp, response}` envelope, diskcache modes, and the cache-key recipe live in the cache crates and only call into this crate +- Intended users + - `cache-response` codec: reading `str(dict)` values Python's sync Redis path writes (`literal_eval`) + - `cache-disk`: diskcache's pickled values (`pickle`), falsy-is-miss (`truthy`) + - Cache-key derivation: sha256 over `str(value)` must match Python byte for byte (`repr::to_str`) + - Byte-identical writes where Python compares raw values (`json::dumps`, `repr`) +- Relation to [`py_literal`](https://docs.rs/py_literal/latest/py_literal/): replaced, do not reintroduce + - Its pest grammar backtracks: parse time doubles per nested `[`/`{` (105 ms at depth 16); ours is linear (19 µs at depth 128) + - Its formatter is not `repr` (`2e-1`, always single quotes, escapes non-ASCII); it also lost `-0.0`, `(1+2j)`, `set()` + - `cache-response` and `cache-disk` still depend on it; migrate them here +- Relation to [`serde-pickle`](https://docs.rs/serde-pickle/latest/serde_pickle/): the pickle codec, used only through its serde interface + - Never `serde_pickle::Value`: its `BTreeMap` dicts reorder keys + - Accepted limits: ints beyond i64, `tuple`/`set`/`frozenset` decode as lists, class references (`GLOBAL`/`REDUCE`) fail; writes protocol 3 +- Every behavior is pinned by CPython output, not by reasoning; everything under `generated/` is script output, never hand-edited + - Regenerate `generated/values.json` with `scripts/generate_fixtures.py`; add a corpus row before changing behavior + - Divergences go in `KNOWN` in `tests/fixtures.rs` with a reason; an entry that starts matching fails until deleted + - Regenerate `generated/nonprintable.rs` with `scripts/generate_nonprintable.py` when the target Python's Unicode version changes + - `scripts/verify_rust_pickles.py` checks CPython reads Rust pickles, with class resolution disabled; CI does not run Python +- Decoders recurse, so they reject nesting beyond `MAX_DEPTH` for stack safety: deliberately stricter than CPython, whose parser takes ~200 levels and whose unpickler has no limit (pinned as `nested_150`) + - Formatters (`repr`, `json`) are unbounded; values from the decoders are already capped, a hand-built `Value` is the caller's responsibility + - `literal_eval` must stay linear in depth: `tests/limits.rs` times the deepest parse, `benches/formats.rs` measures the curve but is manual, since CI runs no Rust bench diff --git a/litellm-rust/crates/python-compat/Cargo.toml b/litellm-rust/crates/python-compat/Cargo.toml new file mode 100644 index 00000000000..2ab6fb29843 --- /dev/null +++ b/litellm-rust/crates/python-compat/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "litellm-python-compat" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "Python data formats (repr, literal_eval, json.dumps, pickle) reproduced for interop with persisted LiteLLM state" + +[dependencies] +num-bigint = "0.4" +num-traits = "0.2" +serde.workspace = true +serde-pickle = "1.2" +serde_json = { workspace = true, features = ["preserve_order"] } +thiserror.workspace = true + +[dev-dependencies] +criterion.workspace = true +hex = "0.4" +rstest.workspace = true + +[[bench]] +name = "formats" +harness = false diff --git a/litellm-rust/crates/python-compat/benches/formats.rs b/litellm-rust/crates/python-compat/benches/formats.rs new file mode 100644 index 00000000000..f0b83cb1658 --- /dev/null +++ b/litellm-rust/crates/python-compat/benches/formats.rs @@ -0,0 +1,111 @@ +//! Throughput of each format on a cached chat completion, and `literal_eval` cost by nesting. +//! +//! Run one group with `cargo bench -p litellm-python-compat -- cached_completion`, and compare +//! against a stored run with `--save-baseline ` / `--baseline `. +//! +//! `literal_eval/nesting` guards against backtracking: the `py_literal` grammar this parser +//! replaced doubled its time per nested `[` or `{` (105 ms at depth 16), so cost must stay +//! linear in depth for every container shape. + +use std::{hint::black_box, time::Duration}; + +use criterion::{ + BatchSize, BenchmarkGroup, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main, + measurement::WallTime, +}; +use litellm_python_compat::{Value, json, literal::literal_eval, pickle, repr::repr}; + +/// `str(entry)` for the `{timestamp, response}` envelope Python's sync Redis path writes. +fn cached_completion() -> String { + let choices: Vec = (0..4) + .map(|index| { + format!( + "{{'finish_reason': 'stop', 'index': {index}, 'message': {{'content': \ + 'Benchmarks compare the same workload under controlled conditions, so a \ + change in time reflects the code rather than the environment. café 日本 \ + {index}', 'role': 'assistant', 'tool_calls': None, 'function_call': None}}, \ + 'logprobs': None}}" + ) + }) + .collect(); + format!( + "{{'timestamp': 1726000000.123, 'response': {{'id': 'chatcmpl-9x1', 'created': \ + 1726000000, 'model': 'gpt-4o-2024-08-06', 'object': 'chat.completion', \ + 'system_fingerprint': 'fp_1', 'choices': [{}], 'usage': {{'completion_tokens': 120, \ + 'prompt_tokens': 42, 'total_tokens': 162, 'completion_tokens_details': None}}}}}}", + choices.join(", ") + ) +} + +/// Every text format, measured against the source bytes it reads or writes. +fn text_formats(group: &mut BenchmarkGroup<'_, WallTime>, text: &str, value: &Value) { + group.throughput(Throughput::Bytes(text.len() as u64)); + group.bench_function("literal_eval", |bencher| { + bencher.iter(|| literal_eval(black_box(text))) + }); + group.bench_function("repr", |bencher| bencher.iter(|| repr(black_box(value)))); + group.bench_function("json_dumps", |bencher| { + bencher.iter(|| json::dumps(black_box(value))) + }); + group.bench_function("to_json", |bencher| { + bencher.iter(|| json::to_json(black_box(value))) + }); +} + +/// Pickle, measured against its own encoding rather than the source text. +fn binary_formats(group: &mut BenchmarkGroup<'_, WallTime>, value: &Value, pickled: &[u8]) { + group.throughput(Throughput::Bytes(pickled.len() as u64)); + group.bench_function("pickle_dumps", |bencher| { + bencher.iter(|| pickle::dumps(black_box(value))) + }); + group.bench_function("pickle_loads", |bencher| { + bencher.iter(|| pickle::loads(black_box(pickled))) + }); +} + +fn formats(c: &mut Criterion) { + let text = cached_completion(); + let value = literal_eval(&text).expect("benchmark payload is a literal"); + let pickled = pickle::dumps(&value).expect("benchmark payload pickles"); + let dumped = json::dumps(&value).expect("benchmark payload is JSON serializable"); + + let mut group = c.benchmark_group("cached_completion"); + text_formats(&mut group, &text, &value); + binary_formats(&mut group, &value, &pickled); + // `from_json` consumes its input, so each iteration gets a freshly parsed one. + group.throughput(Throughput::Bytes(dumped.len() as u64)); + group.bench_function("from_json", |bencher| { + bencher.iter_batched( + || serde_json::from_str::(&dumped).expect("dumps output parses"), + json::from_json, + BatchSize::SmallInput, + ) + }); + group.finish(); +} + +/// One nesting level of each container shape, as `(name, open, close)`. +const SHAPES: [(&str, &str, &str); 3] = [ + ("list", "[", "]"), + ("dict", "{'a': ", "}"), + ("tuple", "(", ",)"), +]; + +fn literal_nesting(c: &mut Criterion) { + let mut group = c.benchmark_group("literal_eval/nesting"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(3)); + for depth in [4, 16, 64, 128] { + for (shape, open, close) in SHAPES { + let text = format!("{}1{}", open.repeat(depth), close.repeat(depth)); + group.throughput(Throughput::Bytes(text.len() as u64)); + group.bench_with_input(BenchmarkId::new(shape, depth), &text, |bencher, text| { + bencher.iter(|| literal_eval(black_box(text))) + }); + } + } + group.finish(); +} + +criterion_group!(benches, formats, literal_nesting); +criterion_main!(benches); diff --git a/litellm-rust/crates/python-compat/generated/nonprintable.rs b/litellm-rust/crates/python-compat/generated/nonprintable.rs new file mode 100644 index 00000000000..a044210f07d --- /dev/null +++ b/litellm-rust/crates/python-compat/generated/nonprintable.rs @@ -0,0 +1,745 @@ +// Generated by scripts/generate_nonprintable.py from Python 3.14.7 +// (Unicode 16.0.0). Do not edit by hand. + +pub(crate) const UNICODE_VERSION: &str = "16.0.0"; + +/// Inclusive code point ranges for which Python's `str.isprintable()` is false. +pub(crate) const NONPRINTABLE: [(u32, u32); 737] = [ + (0x0000, 0x001F), + (0x007F, 0x00A0), + (0x00AD, 0x00AD), + (0x0378, 0x0379), + (0x0380, 0x0383), + (0x038B, 0x038B), + (0x038D, 0x038D), + (0x03A2, 0x03A2), + (0x0530, 0x0530), + (0x0557, 0x0558), + (0x058B, 0x058C), + (0x0590, 0x0590), + (0x05C8, 0x05CF), + (0x05EB, 0x05EE), + (0x05F5, 0x0605), + (0x061C, 0x061C), + (0x06DD, 0x06DD), + (0x070E, 0x070F), + (0x074B, 0x074C), + (0x07B2, 0x07BF), + (0x07FB, 0x07FC), + (0x082E, 0x082F), + (0x083F, 0x083F), + (0x085C, 0x085D), + (0x085F, 0x085F), + (0x086B, 0x086F), + (0x088F, 0x0896), + (0x08E2, 0x08E2), + (0x0984, 0x0984), + (0x098D, 0x098E), + (0x0991, 0x0992), + (0x09A9, 0x09A9), + (0x09B1, 0x09B1), + (0x09B3, 0x09B5), + (0x09BA, 0x09BB), + (0x09C5, 0x09C6), + (0x09C9, 0x09CA), + (0x09CF, 0x09D6), + (0x09D8, 0x09DB), + (0x09DE, 0x09DE), + (0x09E4, 0x09E5), + (0x09FF, 0x0A00), + (0x0A04, 0x0A04), + (0x0A0B, 0x0A0E), + (0x0A11, 0x0A12), + (0x0A29, 0x0A29), + (0x0A31, 0x0A31), + (0x0A34, 0x0A34), + (0x0A37, 0x0A37), + (0x0A3A, 0x0A3B), + (0x0A3D, 0x0A3D), + (0x0A43, 0x0A46), + (0x0A49, 0x0A4A), + (0x0A4E, 0x0A50), + (0x0A52, 0x0A58), + (0x0A5D, 0x0A5D), + (0x0A5F, 0x0A65), + (0x0A77, 0x0A80), + (0x0A84, 0x0A84), + (0x0A8E, 0x0A8E), + (0x0A92, 0x0A92), + (0x0AA9, 0x0AA9), + (0x0AB1, 0x0AB1), + (0x0AB4, 0x0AB4), + (0x0ABA, 0x0ABB), + (0x0AC6, 0x0AC6), + (0x0ACA, 0x0ACA), + (0x0ACE, 0x0ACF), + (0x0AD1, 0x0ADF), + (0x0AE4, 0x0AE5), + (0x0AF2, 0x0AF8), + (0x0B00, 0x0B00), + (0x0B04, 0x0B04), + (0x0B0D, 0x0B0E), + (0x0B11, 0x0B12), + (0x0B29, 0x0B29), + (0x0B31, 0x0B31), + (0x0B34, 0x0B34), + (0x0B3A, 0x0B3B), + (0x0B45, 0x0B46), + (0x0B49, 0x0B4A), + (0x0B4E, 0x0B54), + (0x0B58, 0x0B5B), + (0x0B5E, 0x0B5E), + (0x0B64, 0x0B65), + (0x0B78, 0x0B81), + (0x0B84, 0x0B84), + (0x0B8B, 0x0B8D), + (0x0B91, 0x0B91), + (0x0B96, 0x0B98), + (0x0B9B, 0x0B9B), + (0x0B9D, 0x0B9D), + (0x0BA0, 0x0BA2), + (0x0BA5, 0x0BA7), + (0x0BAB, 0x0BAD), + (0x0BBA, 0x0BBD), + (0x0BC3, 0x0BC5), + (0x0BC9, 0x0BC9), + (0x0BCE, 0x0BCF), + (0x0BD1, 0x0BD6), + (0x0BD8, 0x0BE5), + (0x0BFB, 0x0BFF), + (0x0C0D, 0x0C0D), + (0x0C11, 0x0C11), + (0x0C29, 0x0C29), + (0x0C3A, 0x0C3B), + (0x0C45, 0x0C45), + (0x0C49, 0x0C49), + (0x0C4E, 0x0C54), + (0x0C57, 0x0C57), + (0x0C5B, 0x0C5C), + (0x0C5E, 0x0C5F), + (0x0C64, 0x0C65), + (0x0C70, 0x0C76), + (0x0C8D, 0x0C8D), + (0x0C91, 0x0C91), + (0x0CA9, 0x0CA9), + (0x0CB4, 0x0CB4), + (0x0CBA, 0x0CBB), + (0x0CC5, 0x0CC5), + (0x0CC9, 0x0CC9), + (0x0CCE, 0x0CD4), + (0x0CD7, 0x0CDC), + (0x0CDF, 0x0CDF), + (0x0CE4, 0x0CE5), + (0x0CF0, 0x0CF0), + (0x0CF4, 0x0CFF), + (0x0D0D, 0x0D0D), + (0x0D11, 0x0D11), + (0x0D45, 0x0D45), + (0x0D49, 0x0D49), + (0x0D50, 0x0D53), + (0x0D64, 0x0D65), + (0x0D80, 0x0D80), + (0x0D84, 0x0D84), + (0x0D97, 0x0D99), + (0x0DB2, 0x0DB2), + (0x0DBC, 0x0DBC), + (0x0DBE, 0x0DBF), + (0x0DC7, 0x0DC9), + (0x0DCB, 0x0DCE), + (0x0DD5, 0x0DD5), + (0x0DD7, 0x0DD7), + (0x0DE0, 0x0DE5), + (0x0DF0, 0x0DF1), + (0x0DF5, 0x0E00), + (0x0E3B, 0x0E3E), + (0x0E5C, 0x0E80), + (0x0E83, 0x0E83), + (0x0E85, 0x0E85), + (0x0E8B, 0x0E8B), + (0x0EA4, 0x0EA4), + (0x0EA6, 0x0EA6), + (0x0EBE, 0x0EBF), + (0x0EC5, 0x0EC5), + (0x0EC7, 0x0EC7), + (0x0ECF, 0x0ECF), + (0x0EDA, 0x0EDB), + (0x0EE0, 0x0EFF), + (0x0F48, 0x0F48), + (0x0F6D, 0x0F70), + (0x0F98, 0x0F98), + (0x0FBD, 0x0FBD), + (0x0FCD, 0x0FCD), + (0x0FDB, 0x0FFF), + (0x10C6, 0x10C6), + (0x10C8, 0x10CC), + (0x10CE, 0x10CF), + (0x1249, 0x1249), + (0x124E, 0x124F), + (0x1257, 0x1257), + (0x1259, 0x1259), + (0x125E, 0x125F), + (0x1289, 0x1289), + (0x128E, 0x128F), + (0x12B1, 0x12B1), + (0x12B6, 0x12B7), + (0x12BF, 0x12BF), + (0x12C1, 0x12C1), + (0x12C6, 0x12C7), + (0x12D7, 0x12D7), + (0x1311, 0x1311), + (0x1316, 0x1317), + (0x135B, 0x135C), + (0x137D, 0x137F), + (0x139A, 0x139F), + (0x13F6, 0x13F7), + (0x13FE, 0x13FF), + (0x1680, 0x1680), + (0x169D, 0x169F), + (0x16F9, 0x16FF), + (0x1716, 0x171E), + (0x1737, 0x173F), + (0x1754, 0x175F), + (0x176D, 0x176D), + (0x1771, 0x1771), + (0x1774, 0x177F), + (0x17DE, 0x17DF), + (0x17EA, 0x17EF), + (0x17FA, 0x17FF), + (0x180E, 0x180E), + (0x181A, 0x181F), + (0x1879, 0x187F), + (0x18AB, 0x18AF), + (0x18F6, 0x18FF), + (0x191F, 0x191F), + (0x192C, 0x192F), + (0x193C, 0x193F), + (0x1941, 0x1943), + (0x196E, 0x196F), + (0x1975, 0x197F), + (0x19AC, 0x19AF), + (0x19CA, 0x19CF), + (0x19DB, 0x19DD), + (0x1A1C, 0x1A1D), + (0x1A5F, 0x1A5F), + (0x1A7D, 0x1A7E), + (0x1A8A, 0x1A8F), + (0x1A9A, 0x1A9F), + (0x1AAE, 0x1AAF), + (0x1ACF, 0x1AFF), + (0x1B4D, 0x1B4D), + (0x1BF4, 0x1BFB), + (0x1C38, 0x1C3A), + (0x1C4A, 0x1C4C), + (0x1C8B, 0x1C8F), + (0x1CBB, 0x1CBC), + (0x1CC8, 0x1CCF), + (0x1CFB, 0x1CFF), + (0x1F16, 0x1F17), + (0x1F1E, 0x1F1F), + (0x1F46, 0x1F47), + (0x1F4E, 0x1F4F), + (0x1F58, 0x1F58), + (0x1F5A, 0x1F5A), + (0x1F5C, 0x1F5C), + (0x1F5E, 0x1F5E), + (0x1F7E, 0x1F7F), + (0x1FB5, 0x1FB5), + (0x1FC5, 0x1FC5), + (0x1FD4, 0x1FD5), + (0x1FDC, 0x1FDC), + (0x1FF0, 0x1FF1), + (0x1FF5, 0x1FF5), + (0x1FFF, 0x200F), + (0x2028, 0x202F), + (0x205F, 0x206F), + (0x2072, 0x2073), + (0x208F, 0x208F), + (0x209D, 0x209F), + (0x20C1, 0x20CF), + (0x20F1, 0x20FF), + (0x218C, 0x218F), + (0x242A, 0x243F), + (0x244B, 0x245F), + (0x2B74, 0x2B75), + (0x2B96, 0x2B96), + (0x2CF4, 0x2CF8), + (0x2D26, 0x2D26), + (0x2D28, 0x2D2C), + (0x2D2E, 0x2D2F), + (0x2D68, 0x2D6E), + (0x2D71, 0x2D7E), + (0x2D97, 0x2D9F), + (0x2DA7, 0x2DA7), + (0x2DAF, 0x2DAF), + (0x2DB7, 0x2DB7), + (0x2DBF, 0x2DBF), + (0x2DC7, 0x2DC7), + (0x2DCF, 0x2DCF), + (0x2DD7, 0x2DD7), + (0x2DDF, 0x2DDF), + (0x2E5E, 0x2E7F), + (0x2E9A, 0x2E9A), + (0x2EF4, 0x2EFF), + (0x2FD6, 0x2FEF), + (0x3000, 0x3000), + (0x3040, 0x3040), + (0x3097, 0x3098), + (0x3100, 0x3104), + (0x3130, 0x3130), + (0x318F, 0x318F), + (0x31E6, 0x31EE), + (0x321F, 0x321F), + (0xA48D, 0xA48F), + (0xA4C7, 0xA4CF), + (0xA62C, 0xA63F), + (0xA6F8, 0xA6FF), + (0xA7CE, 0xA7CF), + (0xA7D2, 0xA7D2), + (0xA7D4, 0xA7D4), + (0xA7DD, 0xA7F1), + (0xA82D, 0xA82F), + (0xA83A, 0xA83F), + (0xA878, 0xA87F), + (0xA8C6, 0xA8CD), + (0xA8DA, 0xA8DF), + (0xA954, 0xA95E), + (0xA97D, 0xA97F), + (0xA9CE, 0xA9CE), + (0xA9DA, 0xA9DD), + (0xA9FF, 0xA9FF), + (0xAA37, 0xAA3F), + (0xAA4E, 0xAA4F), + (0xAA5A, 0xAA5B), + (0xAAC3, 0xAADA), + (0xAAF7, 0xAB00), + (0xAB07, 0xAB08), + (0xAB0F, 0xAB10), + (0xAB17, 0xAB1F), + (0xAB27, 0xAB27), + (0xAB2F, 0xAB2F), + (0xAB6C, 0xAB6F), + (0xABEE, 0xABEF), + (0xABFA, 0xABFF), + (0xD7A4, 0xD7AF), + (0xD7C7, 0xD7CA), + (0xD7FC, 0xF8FF), + (0xFA6E, 0xFA6F), + (0xFADA, 0xFAFF), + (0xFB07, 0xFB12), + (0xFB18, 0xFB1C), + (0xFB37, 0xFB37), + (0xFB3D, 0xFB3D), + (0xFB3F, 0xFB3F), + (0xFB42, 0xFB42), + (0xFB45, 0xFB45), + (0xFBC3, 0xFBD2), + (0xFD90, 0xFD91), + (0xFDC8, 0xFDCE), + (0xFDD0, 0xFDEF), + (0xFE1A, 0xFE1F), + (0xFE53, 0xFE53), + (0xFE67, 0xFE67), + (0xFE6C, 0xFE6F), + (0xFE75, 0xFE75), + (0xFEFD, 0xFF00), + (0xFFBF, 0xFFC1), + (0xFFC8, 0xFFC9), + (0xFFD0, 0xFFD1), + (0xFFD8, 0xFFD9), + (0xFFDD, 0xFFDF), + (0xFFE7, 0xFFE7), + (0xFFEF, 0xFFFB), + (0xFFFE, 0xFFFF), + (0x1000C, 0x1000C), + (0x10027, 0x10027), + (0x1003B, 0x1003B), + (0x1003E, 0x1003E), + (0x1004E, 0x1004F), + (0x1005E, 0x1007F), + (0x100FB, 0x100FF), + (0x10103, 0x10106), + (0x10134, 0x10136), + (0x1018F, 0x1018F), + (0x1019D, 0x1019F), + (0x101A1, 0x101CF), + (0x101FE, 0x1027F), + (0x1029D, 0x1029F), + (0x102D1, 0x102DF), + (0x102FC, 0x102FF), + (0x10324, 0x1032C), + (0x1034B, 0x1034F), + (0x1037B, 0x1037F), + (0x1039E, 0x1039E), + (0x103C4, 0x103C7), + (0x103D6, 0x103FF), + (0x1049E, 0x1049F), + (0x104AA, 0x104AF), + (0x104D4, 0x104D7), + (0x104FC, 0x104FF), + (0x10528, 0x1052F), + (0x10564, 0x1056E), + (0x1057B, 0x1057B), + (0x1058B, 0x1058B), + (0x10593, 0x10593), + (0x10596, 0x10596), + (0x105A2, 0x105A2), + (0x105B2, 0x105B2), + (0x105BA, 0x105BA), + (0x105BD, 0x105BF), + (0x105F4, 0x105FF), + (0x10737, 0x1073F), + (0x10756, 0x1075F), + (0x10768, 0x1077F), + (0x10786, 0x10786), + (0x107B1, 0x107B1), + (0x107BB, 0x107FF), + (0x10806, 0x10807), + (0x10809, 0x10809), + (0x10836, 0x10836), + (0x10839, 0x1083B), + (0x1083D, 0x1083E), + (0x10856, 0x10856), + (0x1089F, 0x108A6), + (0x108B0, 0x108DF), + (0x108F3, 0x108F3), + (0x108F6, 0x108FA), + (0x1091C, 0x1091E), + (0x1093A, 0x1093E), + (0x10940, 0x1097F), + (0x109B8, 0x109BB), + (0x109D0, 0x109D1), + (0x10A04, 0x10A04), + (0x10A07, 0x10A0B), + (0x10A14, 0x10A14), + (0x10A18, 0x10A18), + (0x10A36, 0x10A37), + (0x10A3B, 0x10A3E), + (0x10A49, 0x10A4F), + (0x10A59, 0x10A5F), + (0x10AA0, 0x10ABF), + (0x10AE7, 0x10AEA), + (0x10AF7, 0x10AFF), + (0x10B36, 0x10B38), + (0x10B56, 0x10B57), + (0x10B73, 0x10B77), + (0x10B92, 0x10B98), + (0x10B9D, 0x10BA8), + (0x10BB0, 0x10BFF), + (0x10C49, 0x10C7F), + (0x10CB3, 0x10CBF), + (0x10CF3, 0x10CF9), + (0x10D28, 0x10D2F), + (0x10D3A, 0x10D3F), + (0x10D66, 0x10D68), + (0x10D86, 0x10D8D), + (0x10D90, 0x10E5F), + (0x10E7F, 0x10E7F), + (0x10EAA, 0x10EAA), + (0x10EAE, 0x10EAF), + (0x10EB2, 0x10EC1), + (0x10EC5, 0x10EFB), + (0x10F28, 0x10F2F), + (0x10F5A, 0x10F6F), + (0x10F8A, 0x10FAF), + (0x10FCC, 0x10FDF), + (0x10FF7, 0x10FFF), + (0x1104E, 0x11051), + (0x11076, 0x1107E), + (0x110BD, 0x110BD), + (0x110C3, 0x110CF), + (0x110E9, 0x110EF), + (0x110FA, 0x110FF), + (0x11135, 0x11135), + (0x11148, 0x1114F), + (0x11177, 0x1117F), + (0x111E0, 0x111E0), + (0x111F5, 0x111FF), + (0x11212, 0x11212), + (0x11242, 0x1127F), + (0x11287, 0x11287), + (0x11289, 0x11289), + (0x1128E, 0x1128E), + (0x1129E, 0x1129E), + (0x112AA, 0x112AF), + (0x112EB, 0x112EF), + (0x112FA, 0x112FF), + (0x11304, 0x11304), + (0x1130D, 0x1130E), + (0x11311, 0x11312), + (0x11329, 0x11329), + (0x11331, 0x11331), + (0x11334, 0x11334), + (0x1133A, 0x1133A), + (0x11345, 0x11346), + (0x11349, 0x1134A), + (0x1134E, 0x1134F), + (0x11351, 0x11356), + (0x11358, 0x1135C), + (0x11364, 0x11365), + (0x1136D, 0x1136F), + (0x11375, 0x1137F), + (0x1138A, 0x1138A), + (0x1138C, 0x1138D), + (0x1138F, 0x1138F), + (0x113B6, 0x113B6), + (0x113C1, 0x113C1), + (0x113C3, 0x113C4), + (0x113C6, 0x113C6), + (0x113CB, 0x113CB), + (0x113D6, 0x113D6), + (0x113D9, 0x113E0), + (0x113E3, 0x113FF), + (0x1145C, 0x1145C), + (0x11462, 0x1147F), + (0x114C8, 0x114CF), + (0x114DA, 0x1157F), + (0x115B6, 0x115B7), + (0x115DE, 0x115FF), + (0x11645, 0x1164F), + (0x1165A, 0x1165F), + (0x1166D, 0x1167F), + (0x116BA, 0x116BF), + (0x116CA, 0x116CF), + (0x116E4, 0x116FF), + (0x1171B, 0x1171C), + (0x1172C, 0x1172F), + (0x11747, 0x117FF), + (0x1183C, 0x1189F), + (0x118F3, 0x118FE), + (0x11907, 0x11908), + (0x1190A, 0x1190B), + (0x11914, 0x11914), + (0x11917, 0x11917), + (0x11936, 0x11936), + (0x11939, 0x1193A), + (0x11947, 0x1194F), + (0x1195A, 0x1199F), + (0x119A8, 0x119A9), + (0x119D8, 0x119D9), + (0x119E5, 0x119FF), + (0x11A48, 0x11A4F), + (0x11AA3, 0x11AAF), + (0x11AF9, 0x11AFF), + (0x11B0A, 0x11BBF), + (0x11BE2, 0x11BEF), + (0x11BFA, 0x11BFF), + (0x11C09, 0x11C09), + (0x11C37, 0x11C37), + (0x11C46, 0x11C4F), + (0x11C6D, 0x11C6F), + (0x11C90, 0x11C91), + (0x11CA8, 0x11CA8), + (0x11CB7, 0x11CFF), + (0x11D07, 0x11D07), + (0x11D0A, 0x11D0A), + (0x11D37, 0x11D39), + (0x11D3B, 0x11D3B), + (0x11D3E, 0x11D3E), + (0x11D48, 0x11D4F), + (0x11D5A, 0x11D5F), + (0x11D66, 0x11D66), + (0x11D69, 0x11D69), + (0x11D8F, 0x11D8F), + (0x11D92, 0x11D92), + (0x11D99, 0x11D9F), + (0x11DAA, 0x11EDF), + (0x11EF9, 0x11EFF), + (0x11F11, 0x11F11), + (0x11F3B, 0x11F3D), + (0x11F5B, 0x11FAF), + (0x11FB1, 0x11FBF), + (0x11FF2, 0x11FFE), + (0x1239A, 0x123FF), + (0x1246F, 0x1246F), + (0x12475, 0x1247F), + (0x12544, 0x12F8F), + (0x12FF3, 0x12FFF), + (0x13430, 0x1343F), + (0x13456, 0x1345F), + (0x143FB, 0x143FF), + (0x14647, 0x160FF), + (0x1613A, 0x167FF), + (0x16A39, 0x16A3F), + (0x16A5F, 0x16A5F), + (0x16A6A, 0x16A6D), + (0x16ABF, 0x16ABF), + (0x16ACA, 0x16ACF), + (0x16AEE, 0x16AEF), + (0x16AF6, 0x16AFF), + (0x16B46, 0x16B4F), + (0x16B5A, 0x16B5A), + (0x16B62, 0x16B62), + (0x16B78, 0x16B7C), + (0x16B90, 0x16D3F), + (0x16D7A, 0x16E3F), + (0x16E9B, 0x16EFF), + (0x16F4B, 0x16F4E), + (0x16F88, 0x16F8E), + (0x16FA0, 0x16FDF), + (0x16FE5, 0x16FEF), + (0x16FF2, 0x16FFF), + (0x187F8, 0x187FF), + (0x18CD6, 0x18CFE), + (0x18D09, 0x1AFEF), + (0x1AFF4, 0x1AFF4), + (0x1AFFC, 0x1AFFC), + (0x1AFFF, 0x1AFFF), + (0x1B123, 0x1B131), + (0x1B133, 0x1B14F), + (0x1B153, 0x1B154), + (0x1B156, 0x1B163), + (0x1B168, 0x1B16F), + (0x1B2FC, 0x1BBFF), + (0x1BC6B, 0x1BC6F), + (0x1BC7D, 0x1BC7F), + (0x1BC89, 0x1BC8F), + (0x1BC9A, 0x1BC9B), + (0x1BCA0, 0x1CBFF), + (0x1CCFA, 0x1CCFF), + (0x1CEB4, 0x1CEFF), + (0x1CF2E, 0x1CF2F), + (0x1CF47, 0x1CF4F), + (0x1CFC4, 0x1CFFF), + (0x1D0F6, 0x1D0FF), + (0x1D127, 0x1D128), + (0x1D173, 0x1D17A), + (0x1D1EB, 0x1D1FF), + (0x1D246, 0x1D2BF), + (0x1D2D4, 0x1D2DF), + (0x1D2F4, 0x1D2FF), + (0x1D357, 0x1D35F), + (0x1D379, 0x1D3FF), + (0x1D455, 0x1D455), + (0x1D49D, 0x1D49D), + (0x1D4A0, 0x1D4A1), + (0x1D4A3, 0x1D4A4), + (0x1D4A7, 0x1D4A8), + (0x1D4AD, 0x1D4AD), + (0x1D4BA, 0x1D4BA), + (0x1D4BC, 0x1D4BC), + (0x1D4C4, 0x1D4C4), + (0x1D506, 0x1D506), + (0x1D50B, 0x1D50C), + (0x1D515, 0x1D515), + (0x1D51D, 0x1D51D), + (0x1D53A, 0x1D53A), + (0x1D53F, 0x1D53F), + (0x1D545, 0x1D545), + (0x1D547, 0x1D549), + (0x1D551, 0x1D551), + (0x1D6A6, 0x1D6A7), + (0x1D7CC, 0x1D7CD), + (0x1DA8C, 0x1DA9A), + (0x1DAA0, 0x1DAA0), + (0x1DAB0, 0x1DEFF), + (0x1DF1F, 0x1DF24), + (0x1DF2B, 0x1DFFF), + (0x1E007, 0x1E007), + (0x1E019, 0x1E01A), + (0x1E022, 0x1E022), + (0x1E025, 0x1E025), + (0x1E02B, 0x1E02F), + (0x1E06E, 0x1E08E), + (0x1E090, 0x1E0FF), + (0x1E12D, 0x1E12F), + (0x1E13E, 0x1E13F), + (0x1E14A, 0x1E14D), + (0x1E150, 0x1E28F), + (0x1E2AF, 0x1E2BF), + (0x1E2FA, 0x1E2FE), + (0x1E300, 0x1E4CF), + (0x1E4FA, 0x1E5CF), + (0x1E5FB, 0x1E5FE), + (0x1E600, 0x1E7DF), + (0x1E7E7, 0x1E7E7), + (0x1E7EC, 0x1E7EC), + (0x1E7EF, 0x1E7EF), + (0x1E7FF, 0x1E7FF), + (0x1E8C5, 0x1E8C6), + (0x1E8D7, 0x1E8FF), + (0x1E94C, 0x1E94F), + (0x1E95A, 0x1E95D), + (0x1E960, 0x1EC70), + (0x1ECB5, 0x1ED00), + (0x1ED3E, 0x1EDFF), + (0x1EE04, 0x1EE04), + (0x1EE20, 0x1EE20), + (0x1EE23, 0x1EE23), + (0x1EE25, 0x1EE26), + (0x1EE28, 0x1EE28), + (0x1EE33, 0x1EE33), + (0x1EE38, 0x1EE38), + (0x1EE3A, 0x1EE3A), + (0x1EE3C, 0x1EE41), + (0x1EE43, 0x1EE46), + (0x1EE48, 0x1EE48), + (0x1EE4A, 0x1EE4A), + (0x1EE4C, 0x1EE4C), + (0x1EE50, 0x1EE50), + (0x1EE53, 0x1EE53), + (0x1EE55, 0x1EE56), + (0x1EE58, 0x1EE58), + (0x1EE5A, 0x1EE5A), + (0x1EE5C, 0x1EE5C), + (0x1EE5E, 0x1EE5E), + (0x1EE60, 0x1EE60), + (0x1EE63, 0x1EE63), + (0x1EE65, 0x1EE66), + (0x1EE6B, 0x1EE6B), + (0x1EE73, 0x1EE73), + (0x1EE78, 0x1EE78), + (0x1EE7D, 0x1EE7D), + (0x1EE7F, 0x1EE7F), + (0x1EE8A, 0x1EE8A), + (0x1EE9C, 0x1EEA0), + (0x1EEA4, 0x1EEA4), + (0x1EEAA, 0x1EEAA), + (0x1EEBC, 0x1EEEF), + (0x1EEF2, 0x1EFFF), + (0x1F02C, 0x1F02F), + (0x1F094, 0x1F09F), + (0x1F0AF, 0x1F0B0), + (0x1F0C0, 0x1F0C0), + (0x1F0D0, 0x1F0D0), + (0x1F0F6, 0x1F0FF), + (0x1F1AE, 0x1F1E5), + (0x1F203, 0x1F20F), + (0x1F23C, 0x1F23F), + (0x1F249, 0x1F24F), + (0x1F252, 0x1F25F), + (0x1F266, 0x1F2FF), + (0x1F6D8, 0x1F6DB), + (0x1F6ED, 0x1F6EF), + (0x1F6FD, 0x1F6FF), + (0x1F777, 0x1F77A), + (0x1F7DA, 0x1F7DF), + (0x1F7EC, 0x1F7EF), + (0x1F7F1, 0x1F7FF), + (0x1F80C, 0x1F80F), + (0x1F848, 0x1F84F), + (0x1F85A, 0x1F85F), + (0x1F888, 0x1F88F), + (0x1F8AE, 0x1F8AF), + (0x1F8BC, 0x1F8BF), + (0x1F8C2, 0x1F8FF), + (0x1FA54, 0x1FA5F), + (0x1FA6E, 0x1FA6F), + (0x1FA7D, 0x1FA7F), + (0x1FA8A, 0x1FA8E), + (0x1FAC7, 0x1FACD), + (0x1FADD, 0x1FADE), + (0x1FAEA, 0x1FAEF), + (0x1FAF9, 0x1FAFF), + (0x1FB93, 0x1FB93), + (0x1FBFA, 0x1FFFF), + (0x2A6E0, 0x2A6FF), + (0x2B73A, 0x2B73F), + (0x2B81E, 0x2B81F), + (0x2CEA2, 0x2CEAF), + (0x2EBE1, 0x2EBEF), + (0x2EE5E, 0x2F7FF), + (0x2FA1E, 0x2FFFF), + (0x3134B, 0x3134F), + (0x323B0, 0xE00FF), + (0xE01F0, 0x10FFFF), +]; diff --git a/litellm-rust/crates/python-compat/generated/values.json b/litellm-rust/crates/python-compat/generated/values.json new file mode 100644 index 00000000000..7f7e1e35a7d --- /dev/null +++ b/litellm-rust/crates/python-compat/generated/values.json @@ -0,0 +1,2164 @@ +{ + "python": "3.14.7", + "rows": [ + { + "name": "None", + "source": "None", + "literal": true, + "plain": true, + "repr": "None", + "str": "None", + "truthy": false, + "json": "null", + "pickle": { + "0": "4e2e", + "1": "4e2e", + "2": "80024e2e", + "3": "80034e2e", + "4": "80044e2e", + "5": "80054e2e" + }, + "view": "None" + }, + { + "name": "True", + "source": "True", + "literal": true, + "plain": true, + "repr": "True", + "str": "True", + "truthy": true, + "json": "true", + "pickle": { + "0": "4930310a2e", + "1": "4930310a2e", + "2": "8002882e", + "3": "8003882e", + "4": "8004882e", + "5": "8005882e" + }, + "view": "True" + }, + { + "name": "False", + "source": "False", + "literal": true, + "plain": true, + "repr": "False", + "str": "False", + "truthy": false, + "json": "false", + "pickle": { + "0": "4930300a2e", + "1": "4930300a2e", + "2": "8002892e", + "3": "8003892e", + "4": "8004892e", + "5": "8005892e" + }, + "view": "False" + }, + { + "name": "0", + "source": "0", + "literal": true, + "plain": true, + "repr": "0", + "str": "0", + "truthy": false, + "json": "0", + "pickle": { + "0": "49300a2e", + "1": "4b002e", + "2": "80024b002e", + "3": "80034b002e", + "4": "80044b002e", + "5": "80054b002e" + }, + "view": "0" + }, + { + "name": "-7", + "source": "-7", + "literal": true, + "plain": true, + "repr": "-7", + "str": "-7", + "truthy": true, + "json": "-7", + "pickle": { + "0": "492d370a2e", + "1": "4af9ffffff2e", + "2": "80024af9ffffff2e", + "3": "80034af9ffffff2e", + "4": "80049506000000000000004af9ffffff2e", + "5": "80059506000000000000004af9ffffff2e" + }, + "view": "-7" + }, + { + "name": "2**63 - 1", + "source": "2**63 - 1", + "literal": true, + "plain": true, + "repr": "9223372036854775807", + "str": "9223372036854775807", + "truthy": true, + "json": "9223372036854775807", + "pickle": { + "0": "4c393232333337323033363835343737353830374c0a2e", + "1": "4c393232333337323033363835343737353830374c0a2e", + "2": "80028a08ffffffffffffff7f2e", + "3": "80038a08ffffffffffffff7f2e", + "4": "8004950b000000000000008a08ffffffffffffff7f2e", + "5": "8005950b000000000000008a08ffffffffffffff7f2e" + }, + "view": "9223372036854775807" + }, + { + "name": "-(2**63)", + "source": "-(2**63)", + "literal": true, + "plain": true, + "repr": "-9223372036854775808", + "str": "-9223372036854775808", + "truthy": true, + "json": "-9223372036854775808", + "pickle": { + "0": "4c2d393232333337323033363835343737353830384c0a2e", + "1": "4c2d393232333337323033363835343737353830384c0a2e", + "2": "80028a0800000000000000802e", + "3": "80038a0800000000000000802e", + "4": "8004950b000000000000008a0800000000000000802e", + "5": "8005950b000000000000008a0800000000000000802e" + }, + "view": "-9223372036854775808" + }, + { + "name": "2**64", + "source": "2**64", + "literal": true, + "plain": true, + "repr": "18446744073709551616", + "str": "18446744073709551616", + "truthy": true, + "json": "18446744073709551616", + "pickle": { + "0": "4c31383434363734343037333730393535313631364c0a2e", + "1": "4c31383434363734343037333730393535313631364c0a2e", + "2": "80028a090000000000000000012e", + "3": "80038a090000000000000000012e", + "4": "8004950c000000000000008a090000000000000000012e", + "5": "8005950c000000000000008a090000000000000000012e" + }, + "view": "18446744073709551616" + }, + { + "name": "-(2**70)", + "source": "-(2**70)", + "literal": true, + "plain": true, + "repr": "-1180591620717411303424", + "str": "-1180591620717411303424", + "truthy": true, + "json": "-1180591620717411303424", + "pickle": { + "0": "4c2d313138303539313632303731373431313330333432344c0a2e", + "1": "4c2d313138303539313632303731373431313330333432344c0a2e", + "2": "80028a090000000000000000c02e", + "3": "80038a090000000000000000c02e", + "4": "8004950c000000000000008a090000000000000000c02e", + "5": "8005950c000000000000008a090000000000000000c02e" + }, + "view": "-1180591620717411303424" + }, + { + "name": "0.0", + "source": "0.0", + "literal": true, + "plain": true, + "repr": "0.0", + "str": "0.0", + "truthy": false, + "json": "0.0", + "pickle": { + "0": "46302e300a2e", + "1": "4700000000000000002e", + "2": "80024700000000000000002e", + "3": "80034700000000000000002e", + "4": "8004950a000000000000004700000000000000002e", + "5": "8005950a000000000000004700000000000000002e" + }, + "view": "0.0" + }, + { + "name": "-0.0", + "source": "-0.0", + "literal": true, + "plain": true, + "repr": "-0.0", + "str": "-0.0", + "truthy": false, + "json": "-0.0", + "pickle": { + "0": "462d302e300a2e", + "1": "4780000000000000002e", + "2": "80024780000000000000002e", + "3": "80034780000000000000002e", + "4": "8004950a000000000000004780000000000000002e", + "5": "8005950a000000000000004780000000000000002e" + }, + "view": "-0.0" + }, + { + "name": "0.2", + "source": "0.2", + "literal": true, + "plain": true, + "repr": "0.2", + "str": "0.2", + "truthy": true, + "json": "0.2", + "pickle": { + "0": "46302e320a2e", + "1": "473fc999999999999a2e", + "2": "8002473fc999999999999a2e", + "3": "8003473fc999999999999a2e", + "4": "8004950a00000000000000473fc999999999999a2e", + "5": "8005950a00000000000000473fc999999999999a2e" + }, + "view": "0.2" + }, + { + "name": "1.0", + "source": "1.0", + "literal": true, + "plain": true, + "repr": "1.0", + "str": "1.0", + "truthy": true, + "json": "1.0", + "pickle": { + "0": "46312e300a2e", + "1": "473ff00000000000002e", + "2": "8002473ff00000000000002e", + "3": "8003473ff00000000000002e", + "4": "8004950a00000000000000473ff00000000000002e", + "5": "8005950a00000000000000473ff00000000000002e" + }, + "view": "1.0" + }, + { + "name": "-1.5", + "source": "-1.5", + "literal": true, + "plain": true, + "repr": "-1.5", + "str": "-1.5", + "truthy": true, + "json": "-1.5", + "pickle": { + "0": "462d312e350a2e", + "1": "47bff80000000000002e", + "2": "800247bff80000000000002e", + "3": "800347bff80000000000002e", + "4": "8004950a0000000000000047bff80000000000002e", + "5": "8005950a0000000000000047bff80000000000002e" + }, + "view": "-1.5" + }, + { + "name": "0.1 + 0.2", + "source": "0.1 + 0.2", + "literal": true, + "plain": true, + "repr": "0.30000000000000004", + "str": "0.30000000000000004", + "truthy": true, + "json": "0.30000000000000004", + "pickle": { + "0": "46302e33303030303030303030303030303030340a2e", + "1": "473fd33333333333342e", + "2": "8002473fd33333333333342e", + "3": "8003473fd33333333333342e", + "4": "8004950a00000000000000473fd33333333333342e", + "5": "8005950a00000000000000473fd33333333333342e" + }, + "view": "0.30000000000000004" + }, + { + "name": "123456789.123", + "source": "123456789.123", + "literal": true, + "plain": true, + "repr": "123456789.123", + "str": "123456789.123", + "truthy": true, + "json": "123456789.123", + "pickle": { + "0": "463132333435363738392e3132330a2e", + "1": "47419d6f34547df3b62e", + "2": "800247419d6f34547df3b62e", + "3": "800347419d6f34547df3b62e", + "4": "8004950a0000000000000047419d6f34547df3b62e", + "5": "8005950a0000000000000047419d6f34547df3b62e" + }, + "view": "123456789.123" + }, + { + "name": "1e15", + "source": "1e15", + "literal": true, + "plain": true, + "repr": "1000000000000000.0", + "str": "1000000000000000.0", + "truthy": true, + "json": "1000000000000000.0", + "pickle": { + "0": "46313030303030303030303030303030302e300a2e", + "1": "47430c6bf5263400002e", + "2": "800247430c6bf5263400002e", + "3": "800347430c6bf5263400002e", + "4": "8004950a0000000000000047430c6bf5263400002e", + "5": "8005950a0000000000000047430c6bf5263400002e" + }, + "view": "1000000000000000.0" + }, + { + "name": "1e16", + "source": "1e16", + "literal": true, + "plain": true, + "repr": "1e+16", + "str": "1e+16", + "truthy": true, + "json": "1e+16", + "pickle": { + "0": "4631652b31360a2e", + "1": "474341c37937e080002e", + "2": "8002474341c37937e080002e", + "3": "8003474341c37937e080002e", + "4": "8004950a00000000000000474341c37937e080002e", + "5": "8005950a00000000000000474341c37937e080002e" + }, + "view": "1e+16" + }, + { + "name": "1.5e16", + "source": "1.5e16", + "literal": true, + "plain": true, + "repr": "1.5e+16", + "str": "1.5e+16", + "truthy": true, + "json": "1.5e+16", + "pickle": { + "0": "46312e35652b31360a2e", + "1": "47434aa535d3d0c0002e", + "2": "800247434aa535d3d0c0002e", + "3": "800347434aa535d3d0c0002e", + "4": "8004950a0000000000000047434aa535d3d0c0002e", + "5": "8005950a0000000000000047434aa535d3d0c0002e" + }, + "view": "1.5e+16" + }, + { + "name": "9999999999999998.0", + "source": "9999999999999998.0", + "literal": true, + "plain": true, + "repr": "9999999999999998.0", + "str": "9999999999999998.0", + "truthy": true, + "json": "9999999999999998.0", + "pickle": { + "0": "46393939393939393939393939393939382e300a2e", + "1": "474341c37937e07fff2e", + "2": "8002474341c37937e07fff2e", + "3": "8003474341c37937e07fff2e", + "4": "8004950a00000000000000474341c37937e07fff2e", + "5": "8005950a00000000000000474341c37937e07fff2e" + }, + "view": "9999999999999998.0" + }, + { + "name": "0.0001", + "source": "0.0001", + "literal": true, + "plain": true, + "repr": "0.0001", + "str": "0.0001", + "truthy": true, + "json": "0.0001", + "pickle": { + "0": "46302e303030310a2e", + "1": "473f1a36e2eb1c432d2e", + "2": "8002473f1a36e2eb1c432d2e", + "3": "8003473f1a36e2eb1c432d2e", + "4": "8004950a00000000000000473f1a36e2eb1c432d2e", + "5": "8005950a00000000000000473f1a36e2eb1c432d2e" + }, + "view": "0.0001" + }, + { + "name": "1e-05", + "source": "1e-05", + "literal": true, + "plain": true, + "repr": "1e-05", + "str": "1e-05", + "truthy": true, + "json": "1e-05", + "pickle": { + "0": "4631652d30350a2e", + "1": "473ee4f8b588e368f12e", + "2": "8002473ee4f8b588e368f12e", + "3": "8003473ee4f8b588e368f12e", + "4": "8004950a00000000000000473ee4f8b588e368f12e", + "5": "8005950a00000000000000473ee4f8b588e368f12e" + }, + "view": "1e-05" + }, + { + "name": "1.25e-07", + "source": "1.25e-07", + "literal": true, + "plain": true, + "repr": "1.25e-07", + "str": "1.25e-07", + "truthy": true, + "json": "1.25e-07", + "pickle": { + "0": "46312e3235652d30370a2e", + "1": "473e80c6f7a0b5ed8d2e", + "2": "8002473e80c6f7a0b5ed8d2e", + "3": "8003473e80c6f7a0b5ed8d2e", + "4": "8004950a00000000000000473e80c6f7a0b5ed8d2e", + "5": "8005950a00000000000000473e80c6f7a0b5ed8d2e" + }, + "view": "1.25e-07" + }, + { + "name": "5e-324", + "source": "5e-324", + "literal": true, + "plain": true, + "repr": "5e-324", + "str": "5e-324", + "truthy": true, + "json": "5e-324", + "pickle": { + "0": "4635652d3332340a2e", + "1": "4700000000000000012e", + "2": "80024700000000000000012e", + "3": "80034700000000000000012e", + "4": "8004950a000000000000004700000000000000012e", + "5": "8005950a000000000000004700000000000000012e" + }, + "view": "5e-324" + }, + { + "name": "1.7976931348623157e308", + "source": "1.7976931348623157e308", + "literal": true, + "plain": true, + "repr": "1.7976931348623157e+308", + "str": "1.7976931348623157e+308", + "truthy": true, + "json": "1.7976931348623157e+308", + "pickle": { + "0": "46312e37393736393331333438363233313537652b3330380a2e", + "1": "477fefffffffffffff2e", + "2": "8002477fefffffffffffff2e", + "3": "8003477fefffffffffffff2e", + "4": "8004950a00000000000000477fefffffffffffff2e", + "5": "8005950a00000000000000477fefffffffffffff2e" + }, + "view": "1.7976931348623157e+308" + }, + { + "name": "1e22", + "source": "1e22", + "literal": true, + "plain": true, + "repr": "1e+22", + "str": "1e+22", + "truthy": true, + "json": "1e+22", + "pickle": { + "0": "4631652b32320a2e", + "1": "474480f0cf064dd5922e", + "2": "8002474480f0cf064dd5922e", + "3": "8003474480f0cf064dd5922e", + "4": "8004950a00000000000000474480f0cf064dd5922e", + "5": "8005950a00000000000000474480f0cf064dd5922e" + }, + "view": "1e+22" + }, + { + "name": "float('inf')", + "source": "float('inf')", + "literal": false, + "plain": true, + "repr": "inf", + "str": "inf", + "truthy": true, + "json": "Infinity", + "pickle": { + "0": "46696e660a2e", + "1": "477ff00000000000002e", + "2": "8002477ff00000000000002e", + "3": "8003477ff00000000000002e", + "4": "8004950a00000000000000477ff00000000000002e", + "5": "8005950a00000000000000477ff00000000000002e" + }, + "view": "inf" + }, + { + "name": "float('-inf')", + "source": "float('-inf')", + "literal": false, + "plain": true, + "repr": "-inf", + "str": "-inf", + "truthy": true, + "json": "-Infinity", + "pickle": { + "0": "462d696e660a2e", + "1": "47fff00000000000002e", + "2": "800247fff00000000000002e", + "3": "800347fff00000000000002e", + "4": "8004950a0000000000000047fff00000000000002e", + "5": "8005950a0000000000000047fff00000000000002e" + }, + "view": "-inf" + }, + { + "name": "float('nan')", + "source": "float('nan')", + "literal": false, + "plain": true, + "repr": "nan", + "str": "nan", + "truthy": true, + "json": "NaN", + "pickle": { + "0": "466e616e0a2e", + "1": "477ff80000000000002e", + "2": "8002477ff80000000000002e", + "3": "8003477ff80000000000002e", + "4": "8004950a00000000000000477ff80000000000002e", + "5": "8005950a00000000000000477ff80000000000002e" + }, + "view": "nan" + }, + { + "name": "1j", + "source": "1j", + "literal": true, + "plain": false, + "repr": "1j", + "str": "1j", + "truthy": true, + "json_error": "Object of type complex is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a636f6d706c65780a70300a2846302e300a46312e300a7470310a5270320a2e", + "1": "635f5f6275696c74696e5f5f0a636f6d706c65780a710028470000000000000000473ff00000000000007471015271022e", + "2": "8002635f5f6275696c74696e5f5f0a636f6d706c65780a7100470000000000000000473ff00000000000008671015271022e", + "3": "8003636275696c74696e730a636f6d706c65780a7100470000000000000000473ff00000000000008671015271022e", + "4": "8004952e000000000000008c086275696c74696e73948c07636f6d706c6578949394470000000000000000473ff0000000000000869452942e", + "5": "8005952e000000000000008c086275696c74696e73948c07636f6d706c6578949394470000000000000000473ff0000000000000869452942e" + }, + "view": "1j" + }, + { + "name": "-1j", + "source": "-1j", + "literal": false, + "plain": false, + "repr": "(-0-1j)", + "str": "(-0-1j)", + "truthy": true, + "json_error": "Object of type complex is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a636f6d706c65780a70300a28462d302e300a462d312e300a7470310a5270320a2e", + "1": "635f5f6275696c74696e5f5f0a636f6d706c65780a71002847800000000000000047bff00000000000007471015271022e", + "2": "8002635f5f6275696c74696e5f5f0a636f6d706c65780a710047800000000000000047bff00000000000008671015271022e", + "3": "8003636275696c74696e730a636f6d706c65780a710047800000000000000047bff00000000000008671015271022e", + "4": "8004952e000000000000008c086275696c74696e73948c07636f6d706c657894939447800000000000000047bff0000000000000869452942e", + "5": "8005952e000000000000008c086275696c74696e73948c07636f6d706c657894939447800000000000000047bff0000000000000869452942e" + }, + "view": "(-0-1j)" + }, + { + "name": "complex(0, -1)", + "source": "complex(0, -1)", + "literal": false, + "plain": false, + "repr": "-1j", + "str": "-1j", + "truthy": true, + "json_error": "Object of type complex is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a636f6d706c65780a70300a2846302e300a462d312e300a7470310a5270320a2e", + "1": "635f5f6275696c74696e5f5f0a636f6d706c65780a71002847000000000000000047bff00000000000007471015271022e", + "2": "8002635f5f6275696c74696e5f5f0a636f6d706c65780a710047000000000000000047bff00000000000008671015271022e", + "3": "8003636275696c74696e730a636f6d706c65780a710047000000000000000047bff00000000000008671015271022e", + "4": "8004952e000000000000008c086275696c74696e73948c07636f6d706c657894939447000000000000000047bff0000000000000869452942e", + "5": "8005952e000000000000008c086275696c74696e73948c07636f6d706c657894939447000000000000000047bff0000000000000869452942e" + }, + "view": "-1j" + }, + { + "name": "1+2j", + "source": "1+2j", + "literal": true, + "plain": false, + "repr": "(1+2j)", + "str": "(1+2j)", + "truthy": true, + "json_error": "Object of type complex is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a636f6d706c65780a70300a2846312e300a46322e300a7470310a5270320a2e", + "1": "635f5f6275696c74696e5f5f0a636f6d706c65780a710028473ff00000000000004740000000000000007471015271022e", + "2": "8002635f5f6275696c74696e5f5f0a636f6d706c65780a7100473ff00000000000004740000000000000008671015271022e", + "3": "8003636275696c74696e730a636f6d706c65780a7100473ff00000000000004740000000000000008671015271022e", + "4": "8004952e000000000000008c086275696c74696e73948c07636f6d706c6578949394473ff0000000000000474000000000000000869452942e", + "5": "8005952e000000000000008c086275696c74696e73948c07636f6d706c6578949394473ff0000000000000474000000000000000869452942e" + }, + "view": "(1+2j)" + }, + { + "name": "-1.5-0.5j", + "source": "-1.5-0.5j", + "literal": true, + "plain": false, + "repr": "(-1.5-0.5j)", + "str": "(-1.5-0.5j)", + "truthy": true, + "json_error": "Object of type complex is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a636f6d706c65780a70300a28462d312e350a462d302e350a7470310a5270320a2e", + "1": "635f5f6275696c74696e5f5f0a636f6d706c65780a71002847bff800000000000047bfe00000000000007471015271022e", + "2": "8002635f5f6275696c74696e5f5f0a636f6d706c65780a710047bff800000000000047bfe00000000000008671015271022e", + "3": "8003636275696c74696e730a636f6d706c65780a710047bff800000000000047bfe00000000000008671015271022e", + "4": "8004952e000000000000008c086275696c74696e73948c07636f6d706c657894939447bff800000000000047bfe0000000000000869452942e", + "5": "8005952e000000000000008c086275696c74696e73948c07636f6d706c657894939447bff800000000000047bfe0000000000000869452942e" + }, + "view": "(-1.5-0.5j)" + }, + { + "name": "complex(0.0, 1e16)", + "source": "complex(0.0, 1e16)", + "literal": true, + "plain": false, + "repr": "1e+16j", + "str": "1e+16j", + "truthy": true, + "json_error": "Object of type complex is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a636f6d706c65780a70300a2846302e300a4631652b31360a7470310a5270320a2e", + "1": "635f5f6275696c74696e5f5f0a636f6d706c65780a710028470000000000000000474341c37937e080007471015271022e", + "2": "8002635f5f6275696c74696e5f5f0a636f6d706c65780a7100470000000000000000474341c37937e080008671015271022e", + "3": "8003636275696c74696e730a636f6d706c65780a7100470000000000000000474341c37937e080008671015271022e", + "4": "8004952e000000000000008c086275696c74696e73948c07636f6d706c6578949394470000000000000000474341c37937e08000869452942e", + "5": "8005952e000000000000008c086275696c74696e73948c07636f6d706c6578949394470000000000000000474341c37937e08000869452942e" + }, + "view": "1e+16j" + }, + { + "name": "''", + "source": "''", + "literal": true, + "plain": true, + "repr": "''", + "str": "", + "truthy": false, + "json": "\"\"", + "pickle": { + "0": "560a70300a2e", + "1": "580000000071002e", + "2": "8002580000000071002e", + "3": "8003580000000071002e", + "4": "80049504000000000000008c00942e", + "5": "80059504000000000000008c00942e" + }, + "view": "''" + }, + { + "name": "'plain'", + "source": "'plain'", + "literal": true, + "plain": true, + "repr": "'plain'", + "str": "plain", + "truthy": true, + "json": "\"plain\"", + "pickle": { + "0": "56706c61696e0a70300a2e", + "1": "5805000000706c61696e71002e", + "2": "80025805000000706c61696e71002e", + "3": "80035805000000706c61696e71002e", + "4": "80049509000000000000008c05706c61696e942e", + "5": "80059509000000000000008c05706c61696e942e" + }, + "view": "'plain'" + }, + { + "name": "\"it's\"", + "source": "\"it's\"", + "literal": true, + "plain": true, + "repr": "\"it's\"", + "str": "it's", + "truthy": true, + "json": "\"it's\"", + "pickle": { + "0": "56697427730a70300a2e", + "1": "58040000006974277371002e", + "2": "800258040000006974277371002e", + "3": "800358040000006974277371002e", + "4": "80049508000000000000008c0469742773942e", + "5": "80059508000000000000008c0469742773942e" + }, + "view": "\"it's\"" + }, + { + "name": "'say \"hi\"'", + "source": "'say \"hi\"'", + "literal": true, + "plain": true, + "repr": "'say \"hi\"'", + "str": "say \"hi\"", + "truthy": true, + "json": "\"say \\\"hi\\\"\"", + "pickle": { + "0": "5673617920226869220a70300a2e", + "1": "5808000000736179202268692271002e", + "2": "80025808000000736179202268692271002e", + "3": "80035808000000736179202268692271002e", + "4": "8004950c000000000000008c087361792022686922942e", + "5": "8005950c000000000000008c087361792022686922942e" + }, + "view": "'say \"hi\"'" + }, + { + "name": "'both \\' and \"'", + "source": "'both \\' and \"'", + "literal": true, + "plain": true, + "repr": "'both \\' and \"'", + "str": "both ' and \"", + "truthy": true, + "json": "\"both ' and \\\"\"", + "pickle": { + "0": "56626f7468202720616e6420220a70300a2e", + "1": "580c000000626f7468202720616e64202271002e", + "2": "8002580c000000626f7468202720616e64202271002e", + "3": "8003580c000000626f7468202720616e64202271002e", + "4": "80049510000000000000008c0c626f7468202720616e642022942e", + "5": "80059510000000000000008c0c626f7468202720616e642022942e" + }, + "view": "'both \\' and \"'" + }, + { + "name": "'back\\\\slash'", + "source": "'back\\\\slash'", + "literal": true, + "plain": true, + "repr": "'back\\\\slash'", + "str": "back\\slash", + "truthy": true, + "json": "\"back\\\\slash\"", + "pickle": { + "0": "566261636b5c7530303563736c6173680a70300a2e", + "1": "580a0000006261636b5c736c61736871002e", + "2": "8002580a0000006261636b5c736c61736871002e", + "3": "8003580a0000006261636b5c736c61736871002e", + "4": "8004950e000000000000008c0a6261636b5c736c617368942e", + "5": "8005950e000000000000008c0a6261636b5c736c617368942e" + }, + "view": "'back\\\\slash'" + }, + { + "name": "'\\t\\n\\r'", + "source": "'\\t\\n\\r'", + "literal": true, + "plain": true, + "repr": "'\\t\\n\\r'", + "str": "\t\n\r", + "truthy": true, + "json": "\"\\t\\n\\r\"", + "pickle": { + "0": "56095c75303030615c75303030640a70300a2e", + "1": "5803000000090a0d71002e", + "2": "80025803000000090a0d71002e", + "3": "80035803000000090a0d71002e", + "4": "80049507000000000000008c03090a0d942e", + "5": "80059507000000000000008c03090a0d942e" + }, + "view": "'\\t\\n\\r'" + }, + { + "name": "'\\x00\\x1f\\x7f'", + "source": "'\\x00\\x1f\\x7f'", + "literal": true, + "plain": true, + "repr": "'\\x00\\x1f\\x7f'", + "str": "\u0000\u001f\u007f", + "truthy": true, + "json": "\"\\u0000\\u001f\\u007f\"", + "pickle": { + "0": "565c75303030301f7f0a70300a2e", + "1": "5803000000001f7f71002e", + "2": "80025803000000001f7f71002e", + "3": "80035803000000001f7f71002e", + "4": "80049507000000000000008c03001f7f942e", + "5": "80059507000000000000008c03001f7f942e" + }, + "view": "'\\x00\\x1f\\x7f'" + }, + { + "name": "'\\x85\\xa0\\xad'", + "source": "'\\x85\\xa0\\xad'", + "literal": true, + "plain": true, + "repr": "'\\x85\\xa0\\xad'", + "str": "\u0085\u00a0\u00ad", + "truthy": true, + "json": "\"\\u0085\\u00a0\\u00ad\"", + "pickle": { + "0": "5685a0ad0a70300a2e", + "1": "5806000000c285c2a0c2ad71002e", + "2": "80025806000000c285c2a0c2ad71002e", + "3": "80035806000000c285c2a0c2ad71002e", + "4": "8004950a000000000000008c06c285c2a0c2ad942e", + "5": "8005950a000000000000008c06c285c2a0c2ad942e" + }, + "view": "'\\x85\\xa0\\xad'" + }, + { + "name": "'caf\\xe9'", + "source": "'caf\\xe9'", + "literal": true, + "plain": true, + "repr": "'caf\u00e9'", + "str": "caf\u00e9", + "truthy": true, + "json": "\"caf\\u00e9\"", + "pickle": { + "0": "56636166e90a70300a2e", + "1": "5805000000636166c3a971002e", + "2": "80025805000000636166c3a971002e", + "3": "80035805000000636166c3a971002e", + "4": "80049509000000000000008c05636166c3a9942e", + "5": "80059509000000000000008c05636166c3a9942e" + }, + "view": "'caf\u00e9'" + }, + { + "name": "'\\u65e5\\u672c'", + "source": "'\\u65e5\\u672c'", + "literal": true, + "plain": true, + "repr": "'\u65e5\u672c'", + "str": "\u65e5\u672c", + "truthy": true, + "json": "\"\\u65e5\\u672c\"", + "pickle": { + "0": "565c75363565355c75363732630a70300a2e", + "1": "5806000000e697a5e69cac71002e", + "2": "80025806000000e697a5e69cac71002e", + "3": "80035806000000e697a5e69cac71002e", + "4": "8004950a000000000000008c06e697a5e69cac942e", + "5": "8005950a000000000000008c06e697a5e69cac942e" + }, + "view": "'\u65e5\u672c'" + }, + { + "name": "'\\u200b\\u2028\\u3000'", + "source": "'\\u200b\\u2028\\u3000'", + "literal": true, + "plain": true, + "repr": "'\\u200b\\u2028\\u3000'", + "str": "\u200b\u2028\u3000", + "truthy": true, + "json": "\"\\u200b\\u2028\\u3000\"", + "pickle": { + "0": "565c75323030625c75323032385c75333030300a70300a2e", + "1": "5809000000e2808be280a8e3808071002e", + "2": "80025809000000e2808be280a8e3808071002e", + "3": "80035809000000e2808be280a8e3808071002e", + "4": "8004950d000000000000008c09e2808be280a8e38080942e", + "5": "8005950d000000000000008c09e2808be280a8e38080942e" + }, + "view": "'\\u200b\\u2028\\u3000'" + }, + { + "name": "'\\U0001f600'", + "source": "'\\U0001f600'", + "literal": true, + "plain": true, + "repr": "'\ud83d\ude00'", + "str": "\ud83d\ude00", + "truthy": true, + "json": "\"\\ud83d\\ude00\"", + "pickle": { + "0": "565c5530303031663630300a70300a2e", + "1": "5804000000f09f988071002e", + "2": "80025804000000f09f988071002e", + "3": "80035804000000f09f988071002e", + "4": "80049508000000000000008c04f09f9880942e", + "5": "80059508000000000000008c04f09f9880942e" + }, + "view": "'\ud83d\ude00'" + }, + { + "name": "'\\U000e0001\\U0010ffff'", + "source": "'\\U000e0001\\U0010ffff'", + "literal": true, + "plain": true, + "repr": "'\\U000e0001\\U0010ffff'", + "str": "\udb40\udc01\udbff\udfff", + "truthy": true, + "json": "\"\\udb40\\udc01\\udbff\\udfff\"", + "pickle": { + "0": "565c5530303065303030315c5530303130666666660a70300a2e", + "1": "5808000000f3a08081f48fbfbf71002e", + "2": "80025808000000f3a08081f48fbfbf71002e", + "3": "80035808000000f3a08081f48fbfbf71002e", + "4": "8004950c000000000000008c08f3a08081f48fbfbf942e", + "5": "8005950c000000000000008c08f3a08081f48fbfbf942e" + }, + "view": "'\\U000e0001\\U0010ffff'" + }, + { + "name": "'\\b\\f'", + "source": "'\\b\\f'", + "literal": true, + "plain": true, + "repr": "'\\x08\\x0c'", + "str": "\b\f", + "truthy": true, + "json": "\"\\b\\f\"", + "pickle": { + "0": "56080c0a70300a2e", + "1": "5802000000080c71002e", + "2": "80025802000000080c71002e", + "3": "80035802000000080c71002e", + "4": "80049506000000000000008c02080c942e", + "5": "80059506000000000000008c02080c942e" + }, + "view": "'\\x08\\x0c'" + }, + { + "name": "b''", + "source": "b''", + "literal": true, + "plain": true, + "repr": "b''", + "str": "b''", + "truthy": false, + "json_error": "Object of type bytes is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a62797465730a70300a28745270310a2e", + "1": "635f5f6275696c74696e5f5f0a62797465730a7100295271012e", + "2": "8002635f5f6275696c74696e5f5f0a62797465730a7100295271012e", + "3": "8003430071002e", + "4": "80049504000000000000004300942e", + "5": "80059504000000000000004300942e" + }, + "view": "b''" + }, + { + "name": "b'abc'", + "source": "b'abc'", + "literal": true, + "plain": true, + "repr": "b'abc'", + "str": "b'abc'", + "truthy": true, + "json_error": "Object of type bytes is not JSON serializable", + "pickle": { + "0": "635f636f646563730a656e636f64650a70300a28566162630a70310a566c6174696e310a70320a7470330a5270340a2e", + "1": "635f636f646563730a656e636f64650a7100285803000000616263710158060000006c6174696e3171027471035271042e", + "2": "8002635f636f646563730a656e636f64650a71005803000000616263710158060000006c6174696e3171028671035271042e", + "3": "8003430361626371002e", + "4": "80049507000000000000004303616263942e", + "5": "80059507000000000000004303616263942e" + }, + "view": "b'abc'" + }, + { + "name": "b\"a'b\"", + "source": "b\"a'b\"", + "literal": true, + "plain": true, + "repr": "b\"a'b\"", + "str": "b\"a'b\"", + "truthy": true, + "json_error": "Object of type bytes is not JSON serializable", + "pickle": { + "0": "635f636f646563730a656e636f64650a70300a28566127620a70310a566c6174696e310a70320a7470330a5270340a2e", + "1": "635f636f646563730a656e636f64650a7100285803000000612762710158060000006c6174696e3171027471035271042e", + "2": "8002635f636f646563730a656e636f64650a71005803000000612762710158060000006c6174696e3171028671035271042e", + "3": "8003430361276271002e", + "4": "80049507000000000000004303612762942e", + "5": "80059507000000000000004303612762942e" + }, + "view": "b\"a'b\"" + }, + { + "name": "b'a\"b\\'c'", + "source": "b'a\"b\\'c'", + "literal": true, + "plain": true, + "repr": "b'a\"b\\'c'", + "str": "b'a\"b\\'c'", + "truthy": true, + "json_error": "Object of type bytes is not JSON serializable", + "pickle": { + "0": "635f636f646563730a656e636f64650a70300a285661226227630a70310a566c6174696e310a70320a7470330a5270340a2e", + "1": "635f636f646563730a656e636f64650a71002858050000006122622763710158060000006c6174696e3171027471035271042e", + "2": "8002635f636f646563730a656e636f64650a710058050000006122622763710158060000006c6174696e3171028671035271042e", + "3": "80034305612262276371002e", + "4": "800495090000000000000043056122622763942e", + "5": "800595090000000000000043056122622763942e" + }, + "view": "b'a\"b\\'c'" + }, + { + "name": "b'\\x00\\t\\n\\r\\x7f\\x80\\xff'", + "source": "b'\\x00\\t\\n\\r\\x7f\\x80\\xff'", + "literal": true, + "plain": true, + "repr": "b'\\x00\\t\\n\\r\\x7f\\x80\\xff'", + "str": "b'\\x00\\t\\n\\r\\x7f\\x80\\xff'", + "truthy": true, + "json_error": "Object of type bytes is not JSON serializable", + "pickle": { + "0": "635f636f646563730a656e636f64650a70300a28565c7530303030095c75303030615c75303030647f80ff0a70310a566c6174696e310a70320a7470330a5270340a2e", + "1": "635f636f646563730a656e636f64650a710028580900000000090a0d7fc280c3bf710158060000006c6174696e3171027471035271042e", + "2": "8002635f636f646563730a656e636f64650a7100580900000000090a0d7fc280c3bf710158060000006c6174696e3171028671035271042e", + "3": "8003430700090a0d7f80ff71002e", + "4": "8004950b00000000000000430700090a0d7f80ff942e", + "5": "8005950b00000000000000430700090a0d7f80ff942e" + }, + "view": "b'\\x00\\t\\n\\r\\x7f\\x80\\xff'" + }, + { + "name": "[]", + "source": "[]", + "literal": true, + "plain": true, + "repr": "[]", + "str": "[]", + "truthy": false, + "json": "[]", + "pickle": { + "0": "286c70300a2e", + "1": "5d71002e", + "2": "80025d71002e", + "3": "80035d71002e", + "4": "80045d942e", + "5": "80055d942e" + }, + "view": "[]" + }, + { + "name": "[1, 'a', None, True]", + "source": "[1, 'a', None, True]", + "literal": true, + "plain": true, + "repr": "[1, 'a', None, True]", + "str": "[1, 'a', None, True]", + "truthy": true, + "json": "[1, \"a\", null, true]", + "pickle": { + "0": "286c70300a49310a6156610a70310a614e614930310a612e", + "1": "5d7100284b0158010000006171014e4930310a652e", + "2": "80025d7100284b0158010000006171014e88652e", + "3": "80035d7100284b0158010000006171014e88652e", + "4": "8004950d000000000000005d94284b018c0161944e88652e", + "5": "8005950d000000000000005d94284b018c0161944e88652e" + }, + "view": "[1, 'a', None, True]" + }, + { + "name": "()", + "source": "()", + "literal": true, + "plain": true, + "repr": "()", + "str": "()", + "truthy": false, + "json": "[]", + "pickle": { + "0": "28742e", + "1": "292e", + "2": "8002292e", + "3": "8003292e", + "4": "8004292e", + "5": "8005292e" + }, + "view": "[]" + }, + { + "name": "(1,)", + "source": "(1,)", + "literal": true, + "plain": true, + "repr": "(1,)", + "str": "(1,)", + "truthy": true, + "json": "[1]", + "pickle": { + "0": "2849310a7470300a2e", + "1": "284b017471002e", + "2": "80024b018571002e", + "3": "80034b018571002e", + "4": "80049505000000000000004b0185942e", + "5": "80059505000000000000004b0185942e" + }, + "view": "[1]" + }, + { + "name": "(1, (2, 3))", + "source": "(1, (2, 3))", + "literal": true, + "plain": true, + "repr": "(1, (2, 3))", + "str": "(1, (2, 3))", + "truthy": true, + "json": "[1, [2, 3]]", + "pickle": { + "0": "2849310a2849320a49330a7470300a7470310a2e", + "1": "284b01284b024b037471007471012e", + "2": "80024b014b024b038671008671012e", + "3": "80034b014b024b038671008671012e", + "4": "8004950b000000000000004b014b024b03869486942e", + "5": "8005950b000000000000004b014b024b03869486942e" + }, + "view": "[1, [2, 3]]" + }, + { + "name": "{}", + "source": "{}", + "literal": true, + "plain": true, + "repr": "{}", + "str": "{}", + "truthy": false, + "json": "{}", + "pickle": { + "0": "286470300a2e", + "1": "7d71002e", + "2": "80027d71002e", + "3": "80037d71002e", + "4": "80047d942e", + "5": "80057d942e" + }, + "view": "{}" + }, + { + "name": "{'a': 1, 'b': [1.0, 2.5]}", + "source": "{'a': 1, 'b': [1.0, 2.5]}", + "literal": true, + "plain": true, + "repr": "{'a': 1, 'b': [1.0, 2.5]}", + "str": "{'a': 1, 'b': [1.0, 2.5]}", + "truthy": true, + "json": "{\"a\": 1, \"b\": [1.0, 2.5]}", + "pickle": { + "0": "286470300a56610a70310a49310a7356620a70320a286c70330a46312e300a6146322e350a61732e", + "1": "7d71002858010000006171014b0158010000006271025d710328473ff000000000000047400400000000000065752e", + "2": "80027d71002858010000006171014b0158010000006271025d710328473ff000000000000047400400000000000065752e", + "3": "80037d71002858010000006171014b0158010000006271025d710328473ff000000000000047400400000000000065752e", + "4": "80049525000000000000007d94288c0161944b018c0162945d9428473ff000000000000047400400000000000065752e", + "5": "80059525000000000000007d94288c0161944b018c0162945d9428473ff000000000000047400400000000000065752e" + }, + "view": "{'a': 1, 'b': [1.0, 2.5]}" + }, + { + "name": "{'z': 1, 'a': 2, 'm': 3}", + "source": "{'z': 1, 'a': 2, 'm': 3}", + "literal": true, + "plain": true, + "repr": "{'z': 1, 'a': 2, 'm': 3}", + "str": "{'z': 1, 'a': 2, 'm': 3}", + "truthy": true, + "json": "{\"z\": 1, \"a\": 2, \"m\": 3}", + "pickle": { + "0": "286470300a567a0a70310a49310a7356610a70320a49320a73566d0a70330a49330a732e", + "1": "7d71002858010000007a71014b0158010000006171024b0258010000006d71034b03752e", + "2": "80027d71002858010000007a71014b0158010000006171024b0258010000006d71034b03752e", + "3": "80037d71002858010000007a71014b0158010000006171024b0258010000006d71034b03752e", + "4": "80049517000000000000007d94288c017a944b018c0161944b028c016d944b03752e", + "5": "80059517000000000000007d94288c017a944b018c0161944b028c016d944b03752e" + }, + "view": "{'z': 1, 'a': 2, 'm': 3}" + }, + { + "name": "{1: 'int', 2.5: 'float', True: 'bool', None: 'none'}", + "source": "{1: 'int', 2.5: 'float', True: 'bool', None: 'none'}", + "literal": true, + "plain": true, + "repr": "{1: 'bool', 2.5: 'float', None: 'none'}", + "str": "{1: 'bool', 2.5: 'float', None: 'none'}", + "truthy": true, + "json": "{\"1\": \"bool\", \"2.5\": \"float\", \"null\": \"none\"}", + "pickle": { + "0": "286470300a49310a56626f6f6c0a70310a7346322e350a56666c6f61740a70320a734e566e6f6e650a70330a732e", + "1": "7d7100284b015804000000626f6f6c71014740040000000000005805000000666c6f617471024e58040000006e6f6e657103752e", + "2": "80027d7100284b015804000000626f6f6c71014740040000000000005805000000666c6f617471024e58040000006e6f6e657103752e", + "3": "80037d7100284b015804000000626f6f6c71014740040000000000005805000000666c6f617471024e58040000006e6f6e657103752e", + "4": "80049527000000000000007d94284b018c04626f6f6c944740040000000000008c05666c6f6174944e8c046e6f6e6594752e", + "5": "80059527000000000000007d94284b018c04626f6f6c944740040000000000008c05666c6f6174944e8c046e6f6e6594752e" + }, + "view": "{1: 'bool', 2.5: 'float', None: 'none'}" + }, + { + "name": "{(1, 2): 'tuple key'}", + "source": "{(1, 2): 'tuple key'}", + "literal": true, + "plain": true, + "repr": "{(1, 2): 'tuple key'}", + "str": "{(1, 2): 'tuple key'}", + "truthy": true, + "json_error": "keys must be str, int, float, bool or None, not tuple", + "pickle": { + "0": "286470300a2849310a49320a7470310a567475706c65206b65790a70320a732e", + "1": "7d7100284b014b0274710158090000007475706c65206b65797102732e", + "2": "80027d71004b014b0286710158090000007475706c65206b65797102732e", + "3": "80037d71004b014b0286710158090000007475706c65206b65797102732e", + "4": "80049516000000000000007d944b014b0286948c097475706c65206b657994732e", + "5": "80059516000000000000007d944b014b0286948c097475706c65206b657994732e" + }, + "view": "{[1, 2]: 'tuple key'}" + }, + { + "name": "{'nested': {'deeper': {'deepest': [{}]}}}", + "source": "{'nested': {'deeper': {'deepest': [{}]}}}", + "literal": true, + "plain": true, + "repr": "{'nested': {'deeper': {'deepest': [{}]}}}", + "str": "{'nested': {'deeper': {'deepest': [{}]}}}", + "truthy": true, + "json": "{\"nested\": {\"deeper\": {\"deepest\": [{}]}}}", + "pickle": { + "0": "286470300a566e65737465640a70310a286470320a566465657065720a70330a286470340a56646565706573740a70350a286c70360a286470370a617373732e", + "1": "7d710058060000006e657374656471017d7102580600000064656570657271037d710458070000006465657065737471055d71067d7107617373732e", + "2": "80027d710058060000006e657374656471017d7102580600000064656570657271037d710458070000006465657065737471055d71067d7107617373732e", + "3": "80037d710058060000006e657374656471017d7102580600000064656570657271037d710458070000006465657065737471055d71067d7107617373732e", + "4": "8004952b000000000000007d948c066e6573746564947d948c06646565706572947d948c0764656570657374945d947d94617373732e", + "5": "8005952b000000000000007d948c066e6573746564947d948c06646565706572947d948c0764656570657374945d947d94617373732e" + }, + "view": "{'nested': {'deeper': {'deepest': [{}]}}}" + }, + { + "name": "{1}", + "source": "{1}", + "literal": true, + "plain": true, + "repr": "{1}", + "str": "{1}", + "truthy": true, + "json_error": "Object of type set is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a7365740a70300a28286c70310a49310a617470320a5270330a2e", + "1": "635f5f6275696c74696e5f5f0a7365740a7100285d71014b01617471025271032e", + "2": "8002635f5f6275696c74696e5f5f0a7365740a71005d71014b01618571025271032e", + "3": "8003636275696c74696e730a7365740a71005d71014b01618571025271032e", + "4": "80049507000000000000008f94284b01902e", + "5": "80059507000000000000008f94284b01902e" + }, + "view": "[1]" + }, + { + "name": "set()", + "source": "set()", + "literal": true, + "plain": true, + "repr": "set()", + "str": "set()", + "truthy": false, + "json_error": "Object of type set is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a7365740a70300a28286c70310a7470320a5270330a2e", + "1": "635f5f6275696c74696e5f5f0a7365740a7100285d71017471025271032e", + "2": "8002635f5f6275696c74696e5f5f0a7365740a71005d71018571025271032e", + "3": "8003636275696c74696e730a7365740a71005d71018571025271032e", + "4": "80048f942e", + "5": "80058f942e" + }, + "view": "[]" + }, + { + "name": "frozenset({1})", + "source": "frozenset({1})", + "literal": false, + "plain": true, + "repr": "frozenset({1})", + "str": "frozenset({1})", + "truthy": true, + "json_error": "Object of type frozenset is not JSON serializable", + "pickle": { + "0": "635f5f6275696c74696e5f5f0a66726f7a656e7365740a70300a28286c70310a49310a617470320a5270330a2e", + "1": "635f5f6275696c74696e5f5f0a66726f7a656e7365740a7100285d71014b01617471025271032e", + "2": "8002635f5f6275696c74696e5f5f0a66726f7a656e7365740a71005d71014b01618571025271032e", + "3": "8003636275696c74696e730a66726f7a656e7365740a71005d71014b01618571025271032e", + "4": "8004950600000000000000284b0191942e", + "5": "8005950600000000000000284b0191942e" + }, + "view": "[1]" + }, + { + "name": "[[[[[[[[[[1]]]]]]]]]]", + "source": "[[[[[[[[[[1]]]]]]]]]]", + "literal": true, + "plain": true, + "repr": "[[[[[[[[[[1]]]]]]]]]]", + "str": "[[[[[[[[[[1]]]]]]]]]]", + "truthy": true, + "json": "[[[[[[[[[[1]]]]]]]]]]", + "pickle": { + "0": "286c70300a286c70310a286c70320a286c70330a286c70340a286c70350a286c70360a286c70370a286c70380a286c70390a49310a616161616161616161612e", + "1": "5d71005d71015d71025d71035d71045d71055d71065d71075d71085d71094b01616161616161616161612e", + "2": "80025d71005d71015d71025d71035d71045d71055d71065d71075d71085d71094b01616161616161616161612e", + "3": "80035d71005d71015d71025d71035d71045d71055d71065d71075d71085d71094b01616161616161616161612e", + "4": "80049521000000000000005d945d945d945d945d945d945d945d945d945d944b01616161616161616161612e", + "5": "80059521000000000000005d945d945d945d945d945d945d945d945d945d944b01616161616161616161612e" + }, + "view": "[[[[[[[[[[1]]]]]]]]]]" + }, + { + "name": "nested_150", + "source": "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]", + "literal": true, + "plain": true, + "repr": "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]", + "str": "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]", + "truthy": true, + "json": "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]", + "pickle": { + "0": "286c70300a286c70310a286c70320a286c70330a286c70340a286c70350a286c70360a286c70370a286c70380a286c70390a286c7031300a286c7031310a286c7031320a286c7031330a286c7031340a286c7031350a286c7031360a286c7031370a286c7031380a286c7031390a286c7032300a286c7032310a286c7032320a286c7032330a286c7032340a286c7032350a286c7032360a286c7032370a286c7032380a286c7032390a286c7033300a286c7033310a286c7033320a286c7033330a286c7033340a286c7033350a286c7033360a286c7033370a286c7033380a286c7033390a286c7034300a286c7034310a286c7034320a286c7034330a286c7034340a286c7034350a286c7034360a286c7034370a286c7034380a286c7034390a286c7035300a286c7035310a286c7035320a286c7035330a286c7035340a286c7035350a286c7035360a286c7035370a286c7035380a286c7035390a286c7036300a286c7036310a286c7036320a286c7036330a286c7036340a286c7036350a286c7036360a286c7036370a286c7036380a286c7036390a286c7037300a286c7037310a286c7037320a286c7037330a286c7037340a286c7037350a286c7037360a286c7037370a286c7037380a286c7037390a286c7038300a286c7038310a286c7038320a286c7038330a286c7038340a286c7038350a286c7038360a286c7038370a286c7038380a286c7038390a286c7039300a286c7039310a286c7039320a286c7039330a286c7039340a286c7039350a286c7039360a286c7039370a286c7039380a286c7039390a286c703130300a286c703130310a286c703130320a286c703130330a286c703130340a286c703130350a286c703130360a286c703130370a286c703130380a286c703130390a286c703131300a286c703131310a286c703131320a286c703131330a286c703131340a286c703131350a286c703131360a286c703131370a286c703131380a286c703131390a286c703132300a286c703132310a286c703132320a286c703132330a286c703132340a286c703132350a286c703132360a286c703132370a286c703132380a286c703132390a286c703133300a286c703133310a286c703133320a286c703133330a286c703133340a286c703133350a286c703133360a286c703133370a286c703133380a286c703133390a286c703134300a286c703134310a286c703134320a286c703134330a286c703134340a286c703134350a286c703134360a286c703134370a286c703134380a286c703134390a49310a6161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e", + "1": "5d71005d71015d71025d71035d71045d71055d71065d71075d71085d71095d710a5d710b5d710c5d710d5d710e5d710f5d71105d71115d71125d71135d71145d71155d71165d71175d71185d71195d711a5d711b5d711c5d711d5d711e5d711f5d71205d71215d71225d71235d71245d71255d71265d71275d71285d71295d712a5d712b5d712c5d712d5d712e5d712f5d71305d71315d71325d71335d71345d71355d71365d71375d71385d71395d713a5d713b5d713c5d713d5d713e5d713f5d71405d71415d71425d71435d71445d71455d71465d71475d71485d71495d714a5d714b5d714c5d714d5d714e5d714f5d71505d71515d71525d71535d71545d71555d71565d71575d71585d71595d715a5d715b5d715c5d715d5d715e5d715f5d71605d71615d71625d71635d71645d71655d71665d71675d71685d71695d716a5d716b5d716c5d716d5d716e5d716f5d71705d71715d71725d71735d71745d71755d71765d71775d71785d71795d717a5d717b5d717c5d717d5d717e5d717f5d71805d71815d71825d71835d71845d71855d71865d71875d71885d71895d718a5d718b5d718c5d718d5d718e5d718f5d71905d71915d71925d71935d71945d71954b016161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e", + "2": "80025d71005d71015d71025d71035d71045d71055d71065d71075d71085d71095d710a5d710b5d710c5d710d5d710e5d710f5d71105d71115d71125d71135d71145d71155d71165d71175d71185d71195d711a5d711b5d711c5d711d5d711e5d711f5d71205d71215d71225d71235d71245d71255d71265d71275d71285d71295d712a5d712b5d712c5d712d5d712e5d712f5d71305d71315d71325d71335d71345d71355d71365d71375d71385d71395d713a5d713b5d713c5d713d5d713e5d713f5d71405d71415d71425d71435d71445d71455d71465d71475d71485d71495d714a5d714b5d714c5d714d5d714e5d714f5d71505d71515d71525d71535d71545d71555d71565d71575d71585d71595d715a5d715b5d715c5d715d5d715e5d715f5d71605d71615d71625d71635d71645d71655d71665d71675d71685d71695d716a5d716b5d716c5d716d5d716e5d716f5d71705d71715d71725d71735d71745d71755d71765d71775d71785d71795d717a5d717b5d717c5d717d5d717e5d717f5d71805d71815d71825d71835d71845d71855d71865d71875d71885d71895d718a5d718b5d718c5d718d5d718e5d718f5d71905d71915d71925d71935d71945d71954b016161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e", + "3": "80035d71005d71015d71025d71035d71045d71055d71065d71075d71085d71095d710a5d710b5d710c5d710d5d710e5d710f5d71105d71115d71125d71135d71145d71155d71165d71175d71185d71195d711a5d711b5d711c5d711d5d711e5d711f5d71205d71215d71225d71235d71245d71255d71265d71275d71285d71295d712a5d712b5d712c5d712d5d712e5d712f5d71305d71315d71325d71335d71345d71355d71365d71375d71385d71395d713a5d713b5d713c5d713d5d713e5d713f5d71405d71415d71425d71435d71445d71455d71465d71475d71485d71495d714a5d714b5d714c5d714d5d714e5d714f5d71505d71515d71525d71535d71545d71555d71565d71575d71585d71595d715a5d715b5d715c5d715d5d715e5d715f5d71605d71615d71625d71635d71645d71655d71665d71675d71685d71695d716a5d716b5d716c5d716d5d716e5d716f5d71705d71715d71725d71735d71745d71755d71765d71775d71785d71795d717a5d717b5d717c5d717d5d717e5d717f5d71805d71815d71825d71835d71845d71855d71865d71875d71885d71895d718a5d718b5d718c5d718d5d718e5d718f5d71905d71915d71925d71935d71945d71954b016161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e", + "4": "800495c5010000000000005d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d944b016161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e", + "5": "800595c5010000000000005d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d945d944b016161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161612e" + }, + "view": "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]" + }, + { + "name": "{'timestamp': 1726000000.123, 'response': '{\"id\": \"chatcmpl-1\", \"object\": \"chat.completion\"}'}", + "source": "{'timestamp': 1726000000.123, 'response': '{\"id\": \"chatcmpl-1\", \"object\": \"chat.completion\"}'}", + "literal": true, + "plain": true, + "repr": "{'timestamp': 1726000000.123, 'response': '{\"id\": \"chatcmpl-1\", \"object\": \"chat.completion\"}'}", + "str": "{'timestamp': 1726000000.123, 'response': '{\"id\": \"chatcmpl-1\", \"object\": \"chat.completion\"}'}", + "truthy": true, + "json": "{\"timestamp\": 1726000000.123, \"response\": \"{\\\"id\\\": \\\"chatcmpl-1\\\", \\\"object\\\": \\\"chat.completion\\\"}\"}", + "pickle": { + "0": "286470300a5674696d657374616d700a70310a46313732363030303030302e3132330a7356726573706f6e73650a70320a567b226964223a202263686174636d706c2d31222c20226f626a656374223a2022636861742e636f6d706c6574696f6e227d0a70330a732e", + "1": "7d710028580900000074696d657374616d7071014741d9b82ae007df3b5808000000726573706f6e7365710258310000007b226964223a202263686174636d706c2d31222c20226f626a656374223a2022636861742e636f6d706c6574696f6e227d7103752e", + "2": "80027d710028580900000074696d657374616d7071014741d9b82ae007df3b5808000000726573706f6e7365710258310000007b226964223a202263686174636d706c2d31222c20226f626a656374223a2022636861742e636f6d706c6574696f6e227d7103752e", + "3": "80037d710028580900000074696d657374616d7071014741d9b82ae007df3b5808000000726573706f6e7365710258310000007b226964223a202263686174636d706c2d31222c20226f626a656374223a2022636861742e636f6d706c6574696f6e227d7103752e", + "4": "80049559000000000000007d94288c0974696d657374616d70944741d9b82ae007df3b8c08726573706f6e7365948c317b226964223a202263686174636d706c2d31222c20226f626a656374223a2022636861742e636f6d706c6574696f6e227d94752e", + "5": "80059559000000000000007d94288c0974696d657374616d70944741d9b82ae007df3b8c08726573706f6e7365948c317b226964223a202263686174636d706c2d31222c20226f626a656374223a2022636861742e636f6d706c6574696f6e227d94752e" + }, + "view": "{'timestamp': 1726000000.123, 'response': '{\"id\": \"chatcmpl-1\", \"object\": \"chat.completion\"}'}" + }, + { + "name": "{'model': 'gpt-4o', 'messages': [{'role': 'user', 'content': 'hi'}], 'temperature': 0.2, 'stream': False}", + "source": "{'model': 'gpt-4o', 'messages': [{'role': 'user', 'content': 'hi'}], 'temperature': 0.2, 'stream': False}", + "literal": true, + "plain": true, + "repr": "{'model': 'gpt-4o', 'messages': [{'role': 'user', 'content': 'hi'}], 'temperature': 0.2, 'stream': False}", + "str": "{'model': 'gpt-4o', 'messages': [{'role': 'user', 'content': 'hi'}], 'temperature': 0.2, 'stream': False}", + "truthy": true, + "json": "{\"model\": \"gpt-4o\", \"messages\": [{\"role\": \"user\", \"content\": \"hi\"}], \"temperature\": 0.2, \"stream\": false}", + "pickle": { + "0": "286470300a566d6f64656c0a70310a566770742d346f0a70320a73566d657373616765730a70330a286c70340a286470350a56726f6c650a70360a56757365720a70370a7356636f6e74656e740a70380a5668690a70390a7361735674656d70657261747572650a7031300a46302e320a735673747265616d0a7031310a4930300a732e", + "1": "7d71002858050000006d6f64656c710158060000006770742d346f710258080000006d6573736167657371035d71047d7105285804000000726f6c65710658040000007573657271075807000000636f6e74656e7471085802000000686971097561580b00000074656d7065726174757265710a473fc999999999999a580600000073747265616d710b4930300a752e", + "2": "80027d71002858050000006d6f64656c710158060000006770742d346f710258080000006d6573736167657371035d71047d7105285804000000726f6c65710658040000007573657271075807000000636f6e74656e7471085802000000686971097561580b00000074656d7065726174757265710a473fc999999999999a580600000073747265616d710b89752e", + "3": "80037d71002858050000006d6f64656c710158060000006770742d346f710258080000006d6573736167657371035d71047d7105285804000000726f6c65710658040000007573657271075807000000636f6e74656e7471085802000000686971097561580b00000074656d7065726174757265710a473fc999999999999a580600000073747265616d710b89752e", + "4": "80049566000000000000007d94288c056d6f64656c948c066770742d346f948c086d65737361676573945d947d94288c04726f6c65948c0475736572948c07636f6e74656e74948c0268699475618c0b74656d706572617475726594473fc999999999999a8c0673747265616d9489752e", + "5": "80059566000000000000007d94288c056d6f64656c948c066770742d346f948c086d65737361676573945d947d94288c04726f6c65948c0475736572948c07636f6e74656e74948c0268699475618c0b74656d706572617475726594473fc999999999999a8c0673747265616d9489752e" + }, + "view": "{'model': 'gpt-4o', 'messages': [{'role': 'user', 'content': 'hi'}], 'temperature': 0.2, 'stream': False}" + } + ], + "sources": [ + { + "name": "1", + "source": "1", + "repr": "1" + }, + { + "name": " 1", + "source": " 1", + "repr": "1" + }, + { + "name": "\t1", + "source": "\t1", + "repr": "1" + }, + { + "name": "\n1", + "source": "\n1", + "repr": "1" + }, + { + "name": " \n 1", + "source": " \n 1", + "error": "IndentationError" + }, + { + "name": "1\n", + "source": "1\n", + "repr": "1" + }, + { + "name": "1 # comment", + "source": "1 # comment", + "repr": "1" + }, + { + "name": "# c\n1", + "source": "# c\n1", + "repr": "1" + }, + { + "name": "1 \\\n", + "source": "1 \\\n", + "error": "SyntaxError" + }, + { + "name": "1,", + "source": "1,", + "repr": "(1,)" + }, + { + "name": "1, 2", + "source": "1, 2", + "repr": "(1, 2)" + }, + { + "name": "1,\n2", + "source": "1,\n2", + "error": "SyntaxError" + }, + { + "name": "(1,\n2)", + "source": "(1,\n2)", + "repr": "(1, 2)" + }, + { + "name": "[1,\n 2,\n]", + "source": "[1,\n 2,\n]", + "repr": "[1, 2]" + }, + { + "name": "()", + "source": "()", + "repr": "()" + }, + { + "name": "(1)", + "source": "(1)", + "repr": "1" + }, + { + "name": "((1,))", + "source": "((1,))", + "repr": "(1,)" + }, + { + "name": "(,)", + "source": "(,)", + "error": "SyntaxError" + }, + { + "name": "[,]", + "source": "[,]", + "error": "SyntaxError" + }, + { + "name": "{,}", + "source": "{,}", + "error": "SyntaxError" + }, + { + "name": "[1,]", + "source": "[1,]", + "repr": "[1]" + }, + { + "name": "{'a': 1,}", + "source": "{'a': 1,}", + "repr": "{'a': 1}" + }, + { + "name": "{1,}", + "source": "{1,}", + "repr": "{1}" + }, + { + "name": "{'a': 1 'b': 2}", + "source": "{'a': 1 'b': 2}", + "error": "SyntaxError" + }, + { + "name": "{1: 'a', True: 'b'}", + "source": "{1: 'a', True: 'b'}", + "repr": "{1: 'b'}" + }, + { + "name": "{1, True, 1.0}", + "source": "{1, True, 1.0}", + "repr": "{1}" + }, + { + "name": "{(1, 2): 'x', (1.0, 2): 'y'}", + "source": "{(1, 2): 'x', (1.0, 2): 'y'}", + "repr": "{(1, 2): 'y'}" + }, + { + "name": "{[1]: 2}", + "source": "{[1]: 2}", + "error": "TypeError" + }, + { + "name": "{{1}}", + "source": "{{1}}", + "error": "TypeError" + }, + { + "name": "{(1, [2])}", + "source": "{(1, [2])}", + "error": "TypeError" + }, + { + "name": "set()", + "source": "set()", + "repr": "set()" + }, + { + "name": "set( )", + "source": "set( )", + "repr": "set()" + }, + { + "name": "set([1])", + "source": "set([1])", + "error": "ValueError" + }, + { + "name": "frozenset()", + "source": "frozenset()", + "error": "ValueError" + }, + { + "name": "True", + "source": "True", + "repr": "True" + }, + { + "name": "False", + "source": "False", + "repr": "False" + }, + { + "name": "None", + "source": "None", + "repr": "None" + }, + { + "name": "Truex", + "source": "Truex", + "error": "ValueError" + }, + { + "name": "true", + "source": "true", + "error": "ValueError" + }, + { + "name": "...", + "source": "...", + "repr": "Ellipsis" + }, + { + "name": "0", + "source": "0", + "repr": "0" + }, + { + "name": "00", + "source": "00", + "repr": "0" + }, + { + "name": "0_0", + "source": "0_0", + "repr": "0" + }, + { + "name": "01", + "source": "01", + "error": "SyntaxError" + }, + { + "name": "007", + "source": "007", + "error": "SyntaxError" + }, + { + "name": "1_000", + "source": "1_000", + "repr": "1000" + }, + { + "name": "1_", + "source": "1_", + "error": "SyntaxError" + }, + { + "name": "1__0", + "source": "1__0", + "error": "SyntaxError" + }, + { + "name": "_1", + "source": "_1", + "error": "ValueError" + }, + { + "name": "0x1F", + "source": "0x1F", + "repr": "31" + }, + { + "name": "0X_1f", + "source": "0X_1f", + "repr": "31" + }, + { + "name": "0o17", + "source": "0o17", + "repr": "15" + }, + { + "name": "0b101", + "source": "0b101", + "repr": "5" + }, + { + "name": "0b102", + "source": "0b102", + "error": "SyntaxError" + }, + { + "name": "0x", + "source": "0x", + "error": "SyntaxError" + }, + { + "name": "1e3", + "source": "1e3", + "repr": "1000.0" + }, + { + "name": "1E-3", + "source": "1E-3", + "repr": "0.001" + }, + { + "name": "1e", + "source": "1e", + "error": "SyntaxError" + }, + { + "name": "1.e5", + "source": "1.e5", + "repr": "100000.0" + }, + { + "name": ".5", + "source": ".5", + "repr": "0.5" + }, + { + "name": "5.", + "source": "5.", + "repr": "5.0" + }, + { + "name": "1..", + "source": "1..", + "error": "SyntaxError" + }, + { + "name": "1.5.2", + "source": "1.5.2", + "error": "SyntaxError" + }, + { + "name": "1_0.0_1e1_0", + "source": "1_0.0_1e1_0", + "repr": "100100000000.0" + }, + { + "name": "1e999", + "source": "1e999", + "repr": "inf" + }, + { + "name": "-1e999", + "source": "-1e999", + "repr": "-inf" + }, + { + "name": "1j", + "source": "1j", + "repr": "1j" + }, + { + "name": "1.5J", + "source": "1.5J", + "repr": "1.5j" + }, + { + "name": "010j", + "source": "010j", + "repr": "10j" + }, + { + "name": "010.5", + "source": "010.5", + "repr": "10.5" + }, + { + "name": "1a", + "source": "1a", + "error": "SyntaxError" + }, + { + "name": "0x1g", + "source": "0x1g", + "error": "SyntaxError" + }, + { + "name": "-1", + "source": "-1", + "repr": "-1" + }, + { + "name": "+1", + "source": "+1", + "repr": "1" + }, + { + "name": "- 1", + "source": "- 1", + "repr": "-1" + }, + { + "name": "--1", + "source": "--1", + "error": "ValueError" + }, + { + "name": "-+1", + "source": "-+1", + "error": "ValueError" + }, + { + "name": "-(1)", + "source": "-(1)", + "repr": "-1" + }, + { + "name": "-(-1)", + "source": "-(-1)", + "error": "ValueError" + }, + { + "name": "-(1+2j)", + "source": "-(1+2j)", + "error": "ValueError" + }, + { + "name": "-True", + "source": "-True", + "error": "ValueError" + }, + { + "name": "-'a'", + "source": "-'a'", + "error": "ValueError" + }, + { + "name": "-[1]", + "source": "-[1]", + "error": "ValueError" + }, + { + "name": "1+2j", + "source": "1+2j", + "repr": "(1+2j)" + }, + { + "name": "1-2j", + "source": "1-2j", + "repr": "(1-2j)" + }, + { + "name": "1 + 2j", + "source": "1 + 2j", + "repr": "(1+2j)" + }, + { + "name": "(1)+(2j)", + "source": "(1)+(2j)", + "repr": "(1+2j)" + }, + { + "name": "1+2", + "source": "1+2", + "error": "ValueError" + }, + { + "name": "1+2+3j", + "source": "1+2+3j", + "error": "ValueError" + }, + { + "name": "1+-2j", + "source": "1+-2j", + "error": "ValueError" + }, + { + "name": "2j+1", + "source": "2j+1", + "error": "ValueError" + }, + { + "name": "True+1j", + "source": "True+1j", + "error": "ValueError" + }, + { + "name": "1-0j", + "source": "1-0j", + "repr": "(1-0j)" + }, + { + "name": "0.0-0j", + "source": "0.0-0j", + "repr": "-0j" + }, + { + "name": "-0.0+1j", + "source": "-0.0+1j", + "repr": "1j" + }, + { + "name": "-0.0", + "source": "-0.0", + "repr": "-0.0" + }, + { + "name": "-0", + "source": "-0", + "repr": "0" + }, + { + "name": "(-0.0)", + "source": "(-0.0)", + "repr": "-0.0" + }, + { + "name": "[-0.0, (-0.0)]", + "source": "[-0.0, (-0.0)]", + "repr": "[-0.0, -0.0]" + }, + { + "name": "2**3", + "source": "2**3", + "error": "ValueError" + }, + { + "name": "1*2", + "source": "1*2", + "error": "ValueError" + }, + { + "name": "1 if 1 else 2", + "source": "1 if 1 else 2", + "error": "ValueError" + }, + { + "name": "(1,)(2)", + "source": "(1,)(2)", + "error": "ValueError" + }, + { + "name": "''", + "source": "''", + "repr": "''" + }, + { + "name": "\"\"", + "source": "\"\"", + "repr": "''" + }, + { + "name": "'a' 'b'", + "source": "'a' 'b'", + "repr": "'ab'" + }, + { + "name": "'a' \"b\" '''c'''", + "source": "'a' \"b\" '''c'''", + "repr": "'abc'" + }, + { + "name": "'a' b'b'", + "source": "'a' b'b'", + "error": "SyntaxError" + }, + { + "name": "b'a' b'b'", + "source": "b'a' b'b'", + "repr": "b'ab'" + }, + { + "name": "u'x'", + "source": "u'x'", + "repr": "'x'" + }, + { + "name": "U'x'", + "source": "U'x'", + "repr": "'x'" + }, + { + "name": "r'x'", + "source": "r'x'", + "repr": "'x'" + }, + { + "name": "R'x'", + "source": "R'x'", + "repr": "'x'" + }, + { + "name": "b'x'", + "source": "b'x'", + "repr": "b'x'" + }, + { + "name": "B'x'", + "source": "B'x'", + "repr": "b'x'" + }, + { + "name": "br'x'", + "source": "br'x'", + "repr": "b'x'" + }, + { + "name": "Rb'x'", + "source": "Rb'x'", + "repr": "b'x'" + }, + { + "name": "rB'x'", + "source": "rB'x'", + "repr": "b'x'" + }, + { + "name": "ur'x'", + "source": "ur'x'", + "error": "SyntaxError" + }, + { + "name": "bu'x'", + "source": "bu'x'", + "error": "SyntaxError" + }, + { + "name": "f'x'", + "source": "f'x'", + "error": "ValueError" + }, + { + "name": "rf'x'", + "source": "rf'x'", + "error": "ValueError" + }, + { + "name": "'''a\nb'''", + "source": "'''a\nb'''", + "repr": "'a\\nb'" + }, + { + "name": "\"\"\"a\\\"\"\"\"", + "source": "\"\"\"a\\\"\"\"\"", + "repr": "'a\"'" + }, + { + "name": "'a\nb'", + "source": "'a\nb'", + "error": "SyntaxError" + }, + { + "name": "'a\\\nb'", + "source": "'a\\\nb'", + "repr": "'ab'" + }, + { + "name": "r'a\\\nb'", + "source": "r'a\\\nb'", + "repr": "'a\\\\\\nb'" + }, + { + "name": "'unterminated", + "source": "'unterminated", + "error": "SyntaxError" + }, + { + "name": "'\\a\\b\\f\\n\\r\\t\\v'", + "source": "'\\a\\b\\f\\n\\r\\t\\v'", + "repr": "'\\x07\\x08\\x0c\\n\\r\\t\\x0b'" + }, + { + "name": "'\\0\\12\\101\\1011'", + "source": "'\\0\\12\\101\\1011'", + "repr": "'\\x00\\nAA1'" + }, + { + "name": "'\\777'", + "source": "'\\777'", + "repr": "'\u01ff'" + }, + { + "name": "b'\\777'", + "source": "b'\\777'", + "repr": "b'\\xff'" + }, + { + "name": "b'\\400'", + "source": "b'\\400'", + "repr": "b'\\x00'" + }, + { + "name": "'\\x41'", + "source": "'\\x41'", + "repr": "'A'" + }, + { + "name": "'\\x4'", + "source": "'\\x4'", + "error": "SyntaxError" + }, + { + "name": "'\\u00e9'", + "source": "'\\u00e9'", + "repr": "'\u00e9'" + }, + { + "name": "'\\u00e'", + "source": "'\\u00e'", + "error": "SyntaxError" + }, + { + "name": "'\\U0001F600'", + "source": "'\\U0001F600'", + "repr": "'\ud83d\ude00'" + }, + { + "name": "'\\U00110000'", + "source": "'\\U00110000'", + "error": "SyntaxError" + }, + { + "name": "'\\ud800'", + "source": "'\\ud800'", + "repr": "'\\ud800'" + }, + { + "name": "'\\N{BULLET}'", + "source": "'\\N{BULLET}'", + "repr": "'\u2022'" + }, + { + "name": "'\\q'", + "source": "'\\q'", + "repr": "'\\\\q'" + }, + { + "name": "'\\\\'", + "source": "'\\\\'", + "repr": "'\\\\'" + }, + { + "name": "'\\''", + "source": "'\\''", + "repr": "\"'\"" + }, + { + "name": "\"\\\"\"", + "source": "\"\\\"\"", + "repr": "'\"'" + }, + { + "name": "b'\\u0041'", + "source": "b'\\u0041'", + "repr": "b'\\\\u0041'" + }, + { + "name": "b'\\x41\\xff'", + "source": "b'\\x41\\xff'", + "repr": "b'A\\xff'" + }, + { + "name": "b'caf\u00e9'", + "source": "b'caf\u00e9'", + "error": "SyntaxError" + }, + { + "name": "'caf\u00e9'", + "source": "'caf\u00e9'", + "repr": "'caf\u00e9'" + }, + { + "name": "r'\\d'", + "source": "r'\\d'", + "repr": "'\\\\d'" + }, + { + "name": "r'\\''", + "source": "r'\\''", + "repr": "\"\\\\'\"" + }, + { + "name": "rb'\\d'", + "source": "rb'\\d'", + "repr": "b'\\\\d'" + }, + { + "name": "r'\\'", + "source": "r'\\'", + "error": "SyntaxError" + }, + { + "name": "'\u65e5\ud83d\ude00'", + "source": "'\u65e5\ud83d\ude00'", + "repr": "'\u65e5\ud83d\ude00'" + } + ] +} diff --git a/litellm-rust/crates/python-compat/scripts/generate_fixtures.py b/litellm-rust/crates/python-compat/scripts/generate_fixtures.py new file mode 100644 index 00000000000..26456a638f6 --- /dev/null +++ b/litellm-rust/crates/python-compat/scripts/generate_fixtures.py @@ -0,0 +1,352 @@ +"""Regenerate generated/values.json: what CPython produces for each value in CORPUS. + + python scripts/generate_fixtures.py > generated/values.json + +Each row records `repr`, `str`, `json.dumps` (or its error), `bool`, and `pickle.dumps` at +every protocol. `literal` says whether `ast.literal_eval(repr(value))` gives the value back, +which is how Python reads `str(dict)` text back from a cache; the Rust tests reach the other +rows only through pickle. `view` is `repr` of the value as `pickle::loads` decodes it, with +tuples, sets, and frozensets rendered as lists. `sources` records `ast.literal_eval` on raw +source texts: its result, or the exception it raises. +""" + +import ast +import json +import pickle +import sys +import warnings + +# Entries are source texts, or `(name, source)` when the source is too long to read in a +# test report. `name` is what the Rust `KNOWN` table keys on. +CORPUS = [ + # Scalars + "None", + "True", + "False", + "0", + "-7", + "2**63 - 1", + "-(2**63)", + "2**64", + "-(2**70)", + # Floats around CPython's repr thresholds + "0.0", + "-0.0", + "0.2", + "1.0", + "-1.5", + "0.1 + 0.2", + "123456789.123", + "1e15", + "1e16", + "1.5e16", + "9999999999999998.0", + "0.0001", + "1e-05", + "1.25e-07", + "5e-324", + "1.7976931348623157e308", + "1e22", + "float('inf')", + "float('-inf')", + "float('nan')", + # Complex + "1j", + "-1j", + "complex(0, -1)", + "1+2j", + "-1.5-0.5j", + "complex(0.0, 1e16)", + # Strings: quote selection, escapes, printable and non-printable non-ASCII + "''", + "'plain'", + '"it\'s"', + "'say \"hi\"'", + "'both \\' and \"'", + "'back\\\\slash'", + "'\\t\\n\\r'", + "'\\x00\\x1f\\x7f'", + "'\\x85\\xa0\\xad'", + "'caf\\xe9'", + "'\\u65e5\\u672c'", + "'\\u200b\\u2028\\u3000'", + "'\\U0001f600'", + "'\\U000e0001\\U0010ffff'", + "'\\b\\f'", + # Bytes + "b''", + "b'abc'", + 'b"a\'b"', + "b'a\"b\\'c'", + "b'\\x00\\t\\n\\r\\x7f\\x80\\xff'", + # Containers + "[]", + "[1, 'a', None, True]", + "()", + "(1,)", + "(1, (2, 3))", + "{}", + "{'a': 1, 'b': [1.0, 2.5]}", + "{'z': 1, 'a': 2, 'm': 3}", + "{1: 'int', 2.5: 'float', True: 'bool', None: 'none'}", + "{(1, 2): 'tuple key'}", + "{'nested': {'deeper': {'deepest': [{}]}}}", + "{1}", + "set()", + "frozenset({1})", + "[[[[[[[[[[1]]]]]]]]]]", + # Deeper than the Rust decoders allow: CPython's parser accepts ~200 nested brackets + # and its unpickler has no limit, so this row records a deliberate divergence. + ("nested_150", "[" * 150 + "1" + "]" * 150), + # The shape LiteLLM caches + '{\'timestamp\': 1726000000.123, \'response\': \'{"id": "chatcmpl-1", "object": "chat.completion"}\'}', + "{'model': 'gpt-4o', 'messages': [{'role': 'user', 'content': 'hi'}], 'temperature': 0.2, 'stream': False}", +] + +# Source texts for `literal_eval` itself: tokenizer and evaluator edge cases, recorded with +# CPython's result or the exception it raises. Raw strings keep backslashes literal. +SOURCES = [ + # Layout: leading/trailing whitespace, comments, newlines, continuations + "1", + " 1", + "\t1", + "\n1", + " \n 1", + "1\n", + "1 # comment", + "# c\n1", + "1 \\\n", + "1,", + "1, 2", + "1,\n2", + "(1,\n2)", + "[1,\n 2,\n]", + # Containers and grouping + "()", + "(1)", + "((1,))", + "(,)", + "[,]", + "{,}", + "[1,]", + "{'a': 1,}", + "{1,}", + "{'a': 1 'b': 2}", + "{1: 'a', True: 'b'}", + "{1, True, 1.0}", + "{(1, 2): 'x', (1.0, 2): 'y'}", + "{[1]: 2}", + "{{1}}", + "{(1, [2])}", + "set()", + "set( )", + "set([1])", + "frozenset()", + # Names + "True", + "False", + "None", + "Truex", + "true", + "...", + # Integers and floats + "0", + "00", + "0_0", + "01", + "007", + "1_000", + "1_", + "1__0", + "_1", + "0x1F", + "0X_1f", + "0o17", + "0b101", + "0b102", + "0x", + "1e3", + "1E-3", + "1e", + "1.e5", + ".5", + "5.", + "1..", + "1.5.2", + "1_0.0_1e1_0", + "1e999", + "-1e999", + "1j", + "1.5J", + "010j", + "010.5", + "1a", + "0x1g", + # Signs and complex sums + "-1", + "+1", + "- 1", + "--1", + "-+1", + "-(1)", + "-(-1)", + "-(1+2j)", + "-True", + "-'a'", + "-[1]", + "1+2j", + "1-2j", + "1 + 2j", + "(1)+(2j)", + "1+2", + "1+2+3j", + "1+-2j", + "2j+1", + "True+1j", + "1-0j", + "0.0-0j", + "-0.0+1j", + "-0.0", + "-0", + "(-0.0)", + "[-0.0, (-0.0)]", + "2**3", + "1*2", + "1 if 1 else 2", + "(1,)(2)", + # String prefixes, quoting, and concatenation + "''", + '""', + "'a' 'b'", + "'a' \"b\" '''c'''", + "'a' b'b'", + "b'a' b'b'", + "u'x'", + "U'x'", + "r'x'", + "R'x'", + "b'x'", + "B'x'", + "br'x'", + "Rb'x'", + "rB'x'", + "ur'x'", + "bu'x'", + "f'x'", + "rf'x'", + "'''a\nb'''", + '"""a\\""""', + "'a\nb'", + "'a\\\nb'", + "r'a\\\nb'", + "'unterminated", + # Escapes + r"'\a\b\f\n\r\t\v'", + r"'\0\12\101\1011'", + r"'\777'", + r"b'\777'", + r"b'\400'", + r"'\x41'", + r"'\x4'", + r"'\u00e9'", + r"'\u00e'", + r"'\U0001F600'", + r"'\U00110000'", + r"'\ud800'", + r"'\N{BULLET}'", + r"'\q'", + r"'\\'", + r"'\''", + r'"\""', + r"b'\u0041'", + r"b'\x41\xff'", + "b'café'", + "'café'", + r"r'\d'", + r"r'\''", + r"rb'\d'", + r"r'\'", + "'日\U0001f600'", +] + + +def named(entry): + """Split a corpus entry into its report name and its source text.""" + if isinstance(entry, tuple): + return entry + return entry, entry + + +def evaluate(entry): + name, source = named(entry) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + try: + return {"name": name, "source": source, "repr": repr(ast.literal_eval(source))} + except Exception as error: # noqa: BLE001 - recorded, not raised + return {"name": name, "source": source, "error": type(error).__name__} + + +def view(value): + if isinstance(value, (list, tuple, set, frozenset)): + return "[" + ", ".join(view(item) for item in value) + "]" + if isinstance(value, dict): + return "{" + ", ".join(f"{view(key)}: {view(item)}" for key, item in value.items()) + "}" + return repr(value) + + +def plain(value): + """Whether pickle can encode the value without a class reference such as `complex`.""" + if isinstance(value, complex): + return False + if isinstance(value, (list, tuple, set, frozenset)): + return all(plain(item) for item in value) + if isinstance(value, dict): + return all(plain(key) and plain(item) for key, item in value.items()) + return True + + +def is_literal(value): + try: + parsed = ast.literal_eval(repr(value)) + except (ValueError, SyntaxError): + return False + return repr(parsed) == repr(value) + + +def row(entry): + name, source = named(entry) + value = eval(source) + entry = { + "name": name, + "source": source, + "literal": is_literal(value), + "plain": plain(value), + "repr": repr(value), + "str": str(value), + "truthy": bool(value), + } + try: + entry["json"] = json.dumps(value) + except (TypeError, ValueError) as error: + entry["json_error"] = str(error) + try: + entry["pickle"] = {str(protocol): pickle.dumps(value, protocol=protocol).hex() for protocol in range(6)} + entry["view"] = view(value) + except Exception as error: # noqa: BLE001 - recorded, not raised + entry["pickle_error"] = f"{type(error).__name__}: {error}" + return entry + + +if __name__ == "__main__": + json.dump( + { + "python": sys.version.split()[0], + "rows": [row(entry) for entry in CORPUS], + "sources": [evaluate(entry) for entry in SOURCES], + }, + sys.stdout, + indent=2, + ensure_ascii=True, + ) + sys.stdout.write("\n") diff --git a/litellm-rust/crates/python-compat/scripts/generate_nonprintable.py b/litellm-rust/crates/python-compat/scripts/generate_nonprintable.py new file mode 100644 index 00000000000..70d1fbd2dd1 --- /dev/null +++ b/litellm-rust/crates/python-compat/scripts/generate_nonprintable.py @@ -0,0 +1,35 @@ +"""Regenerate generated/nonprintable.rs from this interpreter's `str.isprintable`. + +`repr(str)` escapes exactly the characters for which `str.isprintable()` is false, so the +table must come from the Python version the gateway interoperates with. + + python scripts/generate_nonprintable.py > generated/nonprintable.rs +""" + +import sys +import unicodedata + +ranges = [] +start = None +for code in range(0x110000): + printable = chr(code).isprintable() + if not printable and start is None: + start = code + elif printable and start is not None: + ranges.append((start, code - 1)) + start = None +if start is not None: + ranges.append((start, 0x10FFFF)) + +lines = [ + f"// Generated by scripts/generate_nonprintable.py from Python {sys.version.split()[0]}", + f"// (Unicode {unicodedata.unidata_version}). Do not edit by hand.", + "", + f'pub(crate) const UNICODE_VERSION: &str = "{unicodedata.unidata_version}";', + "", + "/// Inclusive code point ranges for which Python's `str.isprintable()` is false.", + f"pub(crate) const NONPRINTABLE: [(u32, u32); {len(ranges)}] = [", + *(f" (0x{low:04X}, 0x{high:04X})," for low, high in ranges), + "];", +] +sys.stdout.write("\n".join(lines) + "\n") diff --git a/litellm-rust/crates/python-compat/scripts/verify_rust_pickles.py b/litellm-rust/crates/python-compat/scripts/verify_rust_pickles.py new file mode 100644 index 00000000000..793acad61f9 --- /dev/null +++ b/litellm-rust/crates/python-compat/scripts/verify_rust_pickles.py @@ -0,0 +1,43 @@ +"""Check that CPython unpickles what `pickle::dumps` writes, to the value it was given. + + PYTHON_COMPAT_RUST_PICKLES=rust.tsv cargo test -p litellm-python-compat --test fixtures + python scripts/verify_rust_pickles.py rust.tsv + +The rows are plain data by construction, so this refuses to resolve any class rather than +handing file-controlled bytes to an unrestricted `pickle.loads`. +""" + +import ast +import io +import pickle +import sys + + +class PlainDataUnpickler(pickle.Unpickler): + """An unpickler with `GLOBAL`/`REDUCE` disabled, mirroring `pickle::loads` in Rust.""" + + def find_class(self, module, name): + raise pickle.UnpicklingError(f"refusing to resolve {module}.{name}") + + +def loads(data): + return PlainDataUnpickler(io.BytesIO(data)).load() + + +def main(path): + failures = 0 + rows = 0 + with open(path, encoding="utf-8") as lines: + for line in lines: + data, expected = line.rstrip("\n").split("\t", 1) + rows += 1 + actual = repr(loads(bytes.fromhex(data))) + if actual != repr(ast.literal_eval(expected)): + failures += 1 + sys.stdout.write(f"mismatch: expected {expected}, got {actual}\n") + sys.stdout.write(f"{rows} Rust pickles checked, {failures} mismatches\n") + return 1 if failures or not rows else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1])) diff --git a/litellm-rust/crates/python-compat/src/error.rs b/litellm-rust/crates/python-compat/src/error.rs new file mode 100644 index 00000000000..43990eb7602 --- /dev/null +++ b/litellm-rust/crates/python-compat/src/error.rs @@ -0,0 +1,25 @@ +use crate::MAX_DEPTH; + +/// A failure to read or write a Python format. Messages quote CPython's own wording where +/// the Python side raises (`TypeError`, `ValueError`), so callers can log them as is. +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("malformed Python literal at byte {0}")] + InvalidLiteral(usize), + #[error("unhashable type: '{0}'")] + Unhashable(&'static str), + #[error("value nests deeper than {MAX_DEPTH} levels")] + TooDeep, + #[error("invalid pickle: {0}")] + InvalidPickle(String), + #[error("Object of type {0} is not JSON serializable")] + NotJsonSerializable(&'static str), + #[error("keys must be str, int, float, bool or None, not {0}")] + InvalidJsonKey(&'static str), + #[error("Out of range float values are not JSON compliant")] + NonFiniteFloat, + #[error("integer does not fit in the JSON number range")] + IntegerOutOfRange, + #[error("Object of type {0} cannot be pickled as plain data")] + NotPicklable(&'static str), +} diff --git a/litellm-rust/crates/python-compat/src/json.rs b/litellm-rust/crates/python-compat/src/json.rs new file mode 100644 index 00000000000..d3436519ac4 --- /dev/null +++ b/litellm-rust/crates/python-compat/src/json.rs @@ -0,0 +1,168 @@ +//! `json.dumps` with CPython's default options, and the JSON value `json.loads` returns. +//! +//! Defaults are `ensure_ascii=True`, `allow_nan=True`, separators `(", ", ": ")`, and no key +//! sorting. Python's `json.loads` is mapped by [`from_json`]; a `serde_json` number that does +//! not fit `i64` or `u64` arrives as a float, where Python would keep an `int`. + +use std::fmt::Write; + +use serde_json::{Map, Number}; + +use crate::{Error, Value, repr::float_repr}; + +/// `json.dumps(value)`. +pub fn dumps(value: &Value) -> Result { + let mut out = String::new(); + write_value(&mut out, value)?; + Ok(out) +} + +/// The Python value `json.loads` returns for a JSON document. +pub fn from_json(value: serde_json::Value) -> Value { + match value { + serde_json::Value::Null => Value::None, + serde_json::Value::Bool(value) => Value::Bool(value), + serde_json::Value::Number(number) => { + if let Some(value) = number.as_i64() { + Value::Int(value.into()) + } else if let Some(value) = number.as_u64() { + Value::Int(value.into()) + } else { + Value::Float(number.as_f64().unwrap_or(f64::NAN)) + } + } + serde_json::Value::String(text) => Value::Str(text), + serde_json::Value::Array(values) => { + Value::List(values.into_iter().map(from_json).collect()) + } + serde_json::Value::Object(entries) => Value::Dict( + entries + .into_iter() + .map(|(key, value)| (Value::Str(key), from_json(value))) + .collect(), + ), + } +} + +/// `json.loads(json.dumps(value))` as a `serde_json` value: tuples become arrays and dict +/// keys are coerced to strings as `json.dumps` does. Non-finite floats, which Python writes +/// as `NaN` and `Infinity`, have no `serde_json` form and fail with [`Error::NonFiniteFloat`]. +pub fn to_json(value: &Value) -> Result { + Ok(match value { + Value::None => serde_json::Value::Null, + Value::Bool(value) => serde_json::Value::Bool(*value), + Value::Int(value) => { + let text = value.to_string(); + serde_json::Value::Number( + text.parse::() + .map(Number::from) + .or_else(|_| text.parse::().map(Number::from)) + .map_err(|_| Error::IntegerOutOfRange)?, + ) + } + Value::Float(value) => { + serde_json::Value::Number(Number::from_f64(*value).ok_or(Error::NonFiniteFloat)?) + } + Value::Str(text) => serde_json::Value::String(text.clone()), + Value::List(values) | Value::Tuple(values) => { + serde_json::Value::Array(values.iter().map(to_json).collect::, _>>()?) + } + Value::Dict(entries) => serde_json::Value::Object( + entries + .iter() + .map(|(key, value)| Ok((json_key(key)?, to_json(value)?))) + .collect::, Error>>()?, + ), + value @ (Value::Bytes(_) | Value::Set(_) | Value::Complex { .. }) => { + return Err(Error::NotJsonSerializable(value.type_name())); + } + }) +} + +fn write_value(out: &mut String, value: &Value) -> Result<(), Error> { + match value { + Value::None => out.push_str("null"), + Value::Bool(true) => out.push_str("true"), + Value::Bool(false) => out.push_str("false"), + Value::Int(value) => { + let _ = write!(out, "{value}"); + } + Value::Float(value) => out.push_str(&float_text(*value)), + Value::Str(text) => write_string(out, text), + Value::List(values) | Value::Tuple(values) => { + out.push('['); + for (index, value) in values.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + write_value(out, value)?; + } + out.push(']'); + } + Value::Dict(entries) => { + out.push('{'); + for (index, (key, value)) in entries.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + write_string(out, &json_key(key)?); + out.push_str(": "); + write_value(out, value)?; + } + out.push('}'); + } + value @ (Value::Bytes(_) | Value::Set(_) | Value::Complex { .. }) => { + return Err(Error::NotJsonSerializable(value.type_name())); + } + } + Ok(()) +} + +/// `json.encoder.JSONEncoder.iterencode`'s `floatstr` with `allow_nan=True`. +fn float_text(value: f64) -> String { + if value.is_nan() { + "NaN".to_owned() + } else if value.is_infinite() { + if value > 0.0 { "Infinity" } else { "-Infinity" }.to_owned() + } else { + float_repr(value) + } +} + +/// Dict key coercion in `json.dumps`: scalars become their JSON text, other keys fail. +fn json_key(key: &Value) -> Result { + Ok(match key { + Value::Str(text) => text.clone(), + Value::Int(value) => value.to_string(), + Value::Float(value) => float_text(*value), + Value::Bool(true) => "true".to_owned(), + Value::Bool(false) => "false".to_owned(), + Value::None => "null".to_owned(), + key => return Err(Error::InvalidJsonKey(key.type_name())), + }) +} + +/// `py_encode_basestring_ascii`: escape `"`, `\`, control characters, and everything outside +/// printable ASCII as `\uXXXX`, with surrogate pairs above the BMP. +fn write_string(out: &mut String, text: &str) { + out.push('"'); + for ch in text.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '\u{8}' => out.push_str("\\b"), + '\u{c}' => out.push_str("\\f"), + ' '..='~' => out.push(ch), + ch => { + let mut units = [0u16; 2]; + for unit in ch.encode_utf16(&mut units) { + let _ = write!(out, "\\u{unit:04x}"); + } + } + } + } + out.push('"'); +} diff --git a/litellm-rust/crates/python-compat/src/lib.rs b/litellm-rust/crates/python-compat/src/lib.rs new file mode 100644 index 00000000000..b23aff409f2 --- /dev/null +++ b/litellm-rust/crates/python-compat/src/lib.rs @@ -0,0 +1,39 @@ +//! Python data formats reproduced in Rust, for state that Python LiteLLM writes and reads. +//! +//! Each module mirrors one Python operation over plain data values, and its tests replay +//! fixtures generated by that operation in CPython (`scripts/generate_fixtures.py`): +//! +//! | Module | Python operation | +//! |---|---| +//! | [`literal`] | `ast.literal_eval(text)` | +//! | [`repr`] | `repr(value)` and `str(value)` | +//! | [`json`] | `json.dumps(value)`, and `json.loads(json.dumps(value))` as a JSON value | +//! | [`pickle`] | `pickle.loads(data)` and `pickle.dumps(value)` for plain data | +//! | [`truthy`] | `bool(value)` | +//! +//! [`Value`] is the closed data model these formats share. Live Python objects +//! (descriptors, `__bool__`, `__str__`, callbacks) are out of scope: those belong to the +//! PyO3 boundary in `litellm-python-bridge`, which runs the real protocol. +//! +//! Known limits, each pinned by a test: +//! - `set` iteration order follows Python's hash order, which this crate does not model +//! (string hashes are randomized per process). Sets keep their literal order. +//! - `str` values are Rust `String`s, so lone surrogates cannot be represented. +//! - [`pickle::loads`] decodes `tuple`, `set`, and `frozenset` as lists. + +mod error; +pub mod json; +pub mod literal; +pub mod pickle; +pub mod repr; +pub mod truthy; +mod value; + +pub use error::Error; +pub use num_bigint::BigInt; +pub use value::Value; + +/// Nesting limit for the decoders, which recurse. It keeps untrusted persisted data from +/// overflowing the Rust stack, and is deliberately stricter than CPython, whose parser takes +/// about 200 nested brackets and whose unpickler has no limit. +pub const MAX_DEPTH: usize = 128; diff --git a/litellm-rust/crates/python-compat/src/literal.rs b/litellm-rust/crates/python-compat/src/literal.rs new file mode 100644 index 00000000000..12e9036572f --- /dev/null +++ b/litellm-rust/crates/python-compat/src/literal.rs @@ -0,0 +1,745 @@ +//! `ast.literal_eval(text)`, as a single-pass recursive-descent parser. +//! +//! Tokens follow CPython's tokenizer (string prefixes, escapes, implicit concatenation, +//! numeric underscores and radixes, comments, line continuations). Expressions follow +//! `ast.literal_eval`'s evaluator: +//! +//! - one unary `+`/`-`, applied only to a numeric constant (`-(1)` is fine, `--1` is not) +//! - `a + b` / `a - b` only as a signed real plus or minus a complex constant, with 3.14's +//! mixed-mode rules (`1 - 0j` is `(1-0j)`) +//! - parentheses group without making a tuple; `set()` is the only call +//! - dict keys and set members are deduplicated with Python equality (`1 == 1.0 == True`) +//! +//! Not supported, each pinned by a fixture: `\N{NAME}` escapes, `...`, and escapes that +//! produce lone surrogates. + +use std::collections::{HashMap, hash_map::Entry}; + +use num_bigint::BigInt; +use num_traits::{FromPrimitive, ToPrimitive}; + +use crate::{Error, MAX_DEPTH, Value}; + +pub fn literal_eval(text: &str) -> Result { + let mut parser = Parser { + bytes: text.trim_start_matches([' ', '\t']).as_bytes(), + offset: text.len() - text.trim_start_matches([' ', '\t']).len(), + pos: 0, + depth: 0, + brackets: 0, + }; + parser.skip_leading_lines()?; + let value = parser.top_level()?.value; + parser.skip_trivia(true); + if parser.pos != parser.bytes.len() { + return Err(parser.error()); + } + Ok(value) +} + +/// How a parsed term may take part in `+`/`-`, per `ast.literal_eval`'s `_convert_num` +/// (`Constant`) and `_convert_signed_num` (`Signed`). `Other` is any other node. +#[derive(Clone, Copy, PartialEq)] +enum Kind { + Constant, + Signed, + Other, +} + +struct Term { + value: Value, + kind: Kind, +} + +impl Term { + fn other(value: Value) -> Self { + Self { + value, + kind: Kind::Other, + } + } + + fn is_number(&self) -> bool { + matches!( + self.value, + Value::Int(_) | Value::Float(_) | Value::Complex { .. } + ) + } +} + +struct Parser<'a> { + bytes: &'a [u8], + /// Bytes stripped before `bytes` starts, so errors report offsets into the input. + offset: usize, + pos: usize, + depth: usize, + /// Open brackets: newlines are insignificant only inside them. + brackets: usize, +} + +impl Parser<'_> { + fn error(&self) -> Error { + Error::InvalidLiteral(self.offset + self.pos) + } + + fn peek(&self) -> Option { + self.bytes.get(self.pos).copied() + } + + fn peek_at(&self, ahead: usize) -> Option { + self.bytes.get(self.pos + ahead).copied() + } + + fn expect(&mut self, byte: u8) -> Result<(), Error> { + if self.peek() != Some(byte) { + return Err(self.error()); + } + self.pos += 1; + Ok(()) + } + + fn enter(&mut self) -> Result<(), Error> { + self.depth += 1; + if self.depth > MAX_DEPTH { + return Err(Error::TooDeep); + } + Ok(()) + } + + /// Whitespace, comments, and backslash continuations; newlines too when `newlines`. + fn skip_trivia(&mut self, newlines: bool) { + while let Some(byte) = self.peek() { + match byte { + b' ' | b'\t' | b'\x0c' => self.pos += 1, + b'#' => { + while !matches!(self.peek(), None | Some(b'\n' | b'\r')) { + self.pos += 1; + } + } + // A continuation joins two lines; one that ends the input is an EOF error. + b'\\' if matches!(self.peek_at(1), Some(b'\n' | b'\r')) => { + let len = match (self.peek_at(1), self.peek_at(2)) { + (Some(b'\r'), Some(b'\n')) => 3, + _ => 2, + }; + if self.pos + len >= self.bytes.len() { + break; + } + self.pos += len; + } + b'\n' | b'\r' if newlines => self.pos += 1, + _ => break, + } + } + } + + /// Blank and comment-only lines may precede the expression, whose own line must not be + /// indented (CPython raises `IndentationError`). + fn skip_leading_lines(&mut self) -> Result<(), Error> { + loop { + let line_start = self.pos; + self.skip_trivia(false); + match self.peek() { + Some(b'\n' | b'\r') => self.pos += 1, + Some(_) if line_start > 0 && self.pos > line_start => { + self.pos = line_start; + return Err(self.error()); + } + _ => return Ok(()), + } + } + } + + fn at_logical_line_end(&self) -> bool { + matches!(self.peek(), None | Some(b'\n' | b'\r')) + } + + /// The `eval` input: an expression, or a tuple without parentheses. + fn top_level(&mut self) -> Result { + let first = self.expression()?; + self.skip_trivia(false); + if self.peek() != Some(b',') { + return Ok(first); + } + let mut values = vec![first.value]; + while self.peek() == Some(b',') { + self.pos += 1; + self.skip_trivia(false); + if self.at_logical_line_end() { + break; + } + values.push(self.expression()?.value); + self.skip_trivia(false); + } + Ok(Term::other(Value::Tuple(values))) + } + + /// A sum of unary terms, checked as `ast.literal_eval` checks `BinOp`. + fn expression(&mut self) -> Result { + let mut left = self.unary()?; + loop { + self.skip_trivia(self.brackets > 0); + let subtract = match self.peek() { + Some(b'+') => false, + Some(b'-') => true, + _ => return Ok(left), + }; + let at = self.pos; + self.pos += 1; + let right = self.unary()?; + left = complex_sum(left, subtract, right) + .ok_or(Error::InvalidLiteral(self.offset + at))?; + } + } + + fn unary(&mut self) -> Result { + self.skip_trivia(self.brackets > 0); + let negative = match self.peek() { + Some(b'+') => false, + Some(b'-') => true, + _ => return self.primary(), + }; + let at = self.pos; + self.pos += 1; + self.enter()?; + let operand = self.unary()?; + self.depth -= 1; + if operand.kind != Kind::Constant || !operand.is_number() { + return Err(Error::InvalidLiteral(self.offset + at)); + } + let value = if !negative { + operand.value + } else { + match operand.value { + Value::Int(value) => Value::Int(-value), + Value::Float(value) => Value::Float(-value), + Value::Complex { re, im } => Value::Complex { re: -re, im: -im }, + _ => unreachable!("checked numeric above"), + } + }; + Ok(Term { + value, + kind: Kind::Signed, + }) + } + + fn primary(&mut self) -> Result { + match self.peek() { + Some(b'(') => self.parenthesized(), + Some(b'[') => self.list(), + Some(b'{') => self.braced(), + Some(b'0'..=b'9') => self.number(), + Some(b'.') if matches!(self.peek_at(1), Some(b'0'..=b'9')) => self.number(), + Some(b'\'' | b'"') => self.strings(), + Some(byte) if byte.is_ascii_alphabetic() || byte == b'_' => { + if self.string_prefix_len().is_some() { + return self.strings(); + } + self.name() + } + _ => Err(self.error()), + } + } + + fn open(&mut self) -> Result<(), Error> { + self.enter()?; + self.brackets += 1; + self.pos += 1; + Ok(()) + } + + fn close(&mut self, byte: u8) -> Result<(), Error> { + self.skip_trivia(true); + self.expect(byte)?; + self.brackets -= 1; + self.depth -= 1; + Ok(()) + } + + /// Comma-separated expressions up to `close`, with an optional trailing comma. + fn elements(&mut self, close: u8) -> Result, Error> { + let mut values = Vec::new(); + loop { + self.skip_trivia(true); + if self.peek() == Some(close) { + return Ok(values); + } + values.push(self.expression()?.value); + self.skip_trivia(true); + if self.peek() != Some(b',') { + return Ok(values); + } + self.pos += 1; + } + } + + fn parenthesized(&mut self) -> Result { + self.open()?; + self.skip_trivia(true); + if self.peek() == Some(b')') { + self.close(b')')?; + return Ok(Term::other(Value::Tuple(Vec::new()))); + } + let first = self.expression()?; + self.skip_trivia(true); + if self.peek() != Some(b',') { + self.close(b')')?; + return Ok(first); + } + self.pos += 1; + let mut values = vec![first.value]; + values.extend(self.elements(b')')?); + self.close(b')')?; + Ok(Term::other(Value::Tuple(values))) + } + + fn list(&mut self) -> Result { + self.open()?; + let values = self.elements(b']')?; + self.close(b']')?; + Ok(Term::other(Value::List(values))) + } + + fn braced(&mut self) -> Result { + self.open()?; + self.skip_trivia(true); + if self.peek() == Some(b'}') { + self.close(b'}')?; + return Ok(Term::other(Value::Dict(Vec::new()))); + } + let first = self.expression()?.value; + self.skip_trivia(true); + if self.peek() != Some(b':') { + let mut members = UniqueValues::default(); + members.insert(first, None)?; + if self.peek() == Some(b',') { + self.pos += 1; + for member in self.elements(b'}')? { + members.insert(member, None)?; + } + } + self.close(b'}')?; + return Ok(Term::other(Value::Set(members.keys))); + } + let mut entries = UniqueValues::default(); + let mut key = first; + loop { + self.expect(b':')?; + let value = self.expression()?.value; + entries.insert(key, Some(value))?; + self.skip_trivia(true); + if self.peek() != Some(b',') { + break; + } + self.pos += 1; + self.skip_trivia(true); + if self.peek() == Some(b'}') { + break; + } + key = self.expression()?.value; + self.skip_trivia(true); + } + self.close(b'}')?; + Ok(Term::other(Value::Dict(entries.into_entries()))) + } + + fn identifier(&mut self) -> &[u8] { + let start = self.pos; + while matches!(self.peek(), Some(byte) if byte.is_ascii_alphanumeric() || byte == b'_') { + self.pos += 1; + } + &self.bytes[start..self.pos] + } + + fn name(&mut self) -> Result { + let at = self.pos; + let value = match self.identifier() { + b"True" => Value::Bool(true), + b"False" => Value::Bool(false), + b"None" => Value::None, + b"set" => { + self.skip_trivia(self.brackets > 0); + self.expect(b'(')?; + self.skip_trivia(true); + self.expect(b')')?; + return Ok(Term::other(Value::Set(Vec::new()))); + } + _ => return Err(Error::InvalidLiteral(self.offset + at)), + }; + Ok(Term { + value, + kind: Kind::Constant, + }) + } + + /// Digits with single underscores between them, as CPython's `digitpart`. + fn digits(&mut self, radix: u32, out: &mut String) -> Result<(), Error> { + let start = out.len(); + loop { + match self.peek() { + Some(byte) if (byte as char).is_digit(radix) => { + out.push(byte as char); + self.pos += 1; + } + Some(b'_') + if out.len() > start + && matches!(self.peek_at(1), Some(next) if (next as char).is_digit(radix)) => + { + self.pos += 1; + } + _ => break, + } + } + if out.len() == start { + return Err(self.error()); + } + Ok(()) + } + + fn number(&mut self) -> Result { + let start = self.pos; + let radix = match ( + self.peek(), + self.peek_at(1).map(|byte| byte.to_ascii_lowercase()), + ) { + (Some(b'0'), Some(b'x')) => Some(16), + (Some(b'0'), Some(b'o')) => Some(8), + (Some(b'0'), Some(b'b')) => Some(2), + _ => None, + }; + let mut text = String::new(); + let value = if let Some(radix) = radix { + self.pos += 2; + if self.peek() == Some(b'_') { + self.pos += 1; + } + self.digits(radix, &mut text)?; + Value::Int(BigInt::parse_bytes(text.as_bytes(), radix).ok_or(self.error())?) + } else { + let mut is_float = false; + if self.peek() != Some(b'.') { + self.digits(10, &mut text)?; + } + let integer_digits = text.clone(); + if self.peek() == Some(b'.') { + is_float = true; + self.pos += 1; + text.push('.'); + if matches!(self.peek(), Some(b'0'..=b'9')) { + self.digits(10, &mut text)?; + } + } + if matches!(self.peek(), Some(b'e' | b'E')) { + is_float = true; + self.pos += 1; + text.push('e'); + if let Some(sign @ (b'+' | b'-')) = self.peek() { + text.push(sign as char); + self.pos += 1; + } + self.digits(10, &mut text)?; + } + if matches!(self.peek(), Some(b'j' | b'J')) { + self.pos += 1; + let im = text.parse::().map_err(|_| self.error())?; + Value::Complex { re: 0.0, im } + } else if is_float { + Value::Float(text.parse::().map_err(|_| self.error())?) + } else { + if integer_digits.len() > 1 + && integer_digits.starts_with('0') + && integer_digits.bytes().any(|digit| digit != b'0') + { + return Err(Error::InvalidLiteral(self.offset + start)); + } + Value::Int(BigInt::parse_bytes(integer_digits.as_bytes(), 10).ok_or(self.error())?) + } + }; + if matches!(self.peek(), Some(byte) if byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'.') + { + return Err(self.error()); + } + Ok(Term { + value, + kind: Kind::Constant, + }) + } + + /// The length of a valid string prefix (`r`, `u`, `b`, `br`, `rb`, any case) directly + /// followed by a quote. + fn string_prefix_len(&self) -> Option { + let mut len = 0; + while matches!(self.peek_at(len), Some(byte) if byte.is_ascii_alphabetic()) && len < 3 { + len += 1; + } + if !matches!(self.peek_at(len), Some(b'\'' | b'"')) { + return None; + } + let prefix: Vec = self.bytes[self.pos..self.pos + len] + .iter() + .map(u8::to_ascii_lowercase) + .collect(); + matches!(prefix.as_slice(), b"" | b"r" | b"u" | b"b" | b"br" | b"rb").then_some(len) + } + + /// Adjacent string literals concatenate; `str` and `bytes` cannot mix. + fn strings(&mut self) -> Result { + let mut text: Option = None; + let mut bytes: Option> = None; + loop { + let at = self.pos; + let Some(prefix_len) = self.string_prefix_len() else { + break; + }; + let prefix = &self.bytes[self.pos..self.pos + prefix_len]; + let raw = prefix.iter().any(|byte| byte.eq_ignore_ascii_case(&b'r')); + let is_bytes = prefix.iter().any(|byte| byte.eq_ignore_ascii_case(&b'b')); + self.pos += prefix_len; + let mut out = Vec::new(); + self.string_body(raw, is_bytes, &mut out)?; + if is_bytes { + if text.is_some() { + return Err(Error::InvalidLiteral(self.offset + at)); + } + bytes.get_or_insert_with(Vec::new).extend(out); + } else { + if bytes.is_some() { + return Err(Error::InvalidLiteral(self.offset + at)); + } + let piece = + String::from_utf8(out).map_err(|_| Error::InvalidLiteral(self.offset + at))?; + text.get_or_insert_with(String::new).push_str(&piece); + } + self.skip_trivia(self.brackets > 0); + } + let value = match (text, bytes) { + (Some(text), None) => Value::Str(text), + (None, Some(bytes)) => Value::Bytes(bytes), + _ => return Err(self.error()), + }; + Ok(Term { + value, + kind: Kind::Constant, + }) + } + + /// One quoted body, decoded into UTF-8 (`str`) or raw bytes (`bytes`). + fn string_body(&mut self, raw: bool, is_bytes: bool, out: &mut Vec) -> Result<(), Error> { + let quote = self.bytes[self.pos]; + let triple = self.peek_at(1) == Some(quote) && self.peek_at(2) == Some(quote); + self.pos += if triple { 3 } else { 1 }; + loop { + let Some(byte) = self.peek() else { + return Err(self.error()); + }; + if byte == quote + && (!triple || (self.peek_at(1) == Some(quote) && self.peek_at(2) == Some(quote))) + { + self.pos += if triple { 3 } else { 1 }; + return Ok(()); + } + match byte { + b'\n' | b'\r' if !triple => return Err(self.error()), + b'\\' if raw => { + let Some(next) = self.peek_at(1) else { + return Err(self.error()); + }; + out.push(b'\\'); + self.pos += 1; + if next == b'\n' || next == b'\r' || next == quote || next == b'\\' { + out.push(next); + self.pos += 1; + } + } + b'\\' => { + self.pos += 1; + self.escape(is_bytes, out)?; + } + byte if is_bytes && !byte.is_ascii() => return Err(self.error()), + byte => { + out.push(byte); + self.pos += 1; + } + } + } + } + + fn escape(&mut self, is_bytes: bool, out: &mut Vec) -> Result<(), Error> { + let Some(byte) = self.peek() else { + return Err(self.error()); + }; + self.pos += 1; + let simple = match byte { + b'\n' => return Ok(()), + b'\r' => { + if self.peek() == Some(b'\n') { + self.pos += 1; + } + return Ok(()); + } + b'\\' | b'\'' | b'"' => byte, + b'a' => 0x07, + b'b' => 0x08, + b'f' => 0x0c, + b'n' => b'\n', + b'r' => b'\r', + b't' => b'\t', + b'v' => 0x0b, + b'0'..=b'7' => { + let mut code = u32::from(byte - b'0'); + for _ in 0..2 { + match self.peek() { + Some(digit @ b'0'..=b'7') => { + code = code * 8 + u32::from(digit - b'0'); + self.pos += 1; + } + _ => break, + } + } + // Bytes keep the low eight bits of `\400`-`\777`, as CPython does. + return self.push_code(if is_bytes { code & 0xff } else { code }, is_bytes, out); + } + b'x' => { + let code = self.hex(2)?; + return self.push_code(code, is_bytes, out); + } + b'u' if !is_bytes => { + let code = self.hex(4)?; + return self.push_code(code, is_bytes, out); + } + b'U' if !is_bytes => { + let code = self.hex(8)?; + return self.push_code(code, is_bytes, out); + } + b'N' if !is_bytes => return Err(self.error()), + _ => { + // Unknown escapes keep the backslash (a `SyntaxWarning` in CPython). + out.push(b'\\'); + self.pos -= 1; + return Ok(()); + } + }; + out.push(simple); + Ok(()) + } + + fn hex(&mut self, count: usize) -> Result { + let mut code = 0u32; + for _ in 0..count { + let digit = self + .peek() + .and_then(|byte| (byte as char).to_digit(16)) + .ok_or(self.error())?; + code = code * 16 + digit; + self.pos += 1; + } + Ok(code) + } + + fn push_code(&self, code: u32, is_bytes: bool, out: &mut Vec) -> Result<(), Error> { + if is_bytes { + out.push(u8::try_from(code).map_err(|_| self.error())?); + return Ok(()); + } + let ch = char::from_u32(code).ok_or(self.error())?; + let mut buffer = [0u8; 4]; + out.extend_from_slice(ch.encode_utf8(&mut buffer).as_bytes()); + Ok(()) + } +} + +/// `left + right` or `left - right` as `ast.literal_eval` permits: a signed real on the +/// left and an unsigned complex constant on the right, combined with CPython 3.14's +/// mixed-mode rules, which leave the imaginary part untouched by the real operand. +fn complex_sum(left: Term, subtract: bool, right: Term) -> Option { + if left.kind == Kind::Other || right.kind != Kind::Constant { + return None; + } + let real = match &left.value { + Value::Int(value) => value.to_f64().filter(|value| value.is_finite())?, + Value::Float(value) => *value, + _ => return None, + }; + let Value::Complex { re, im } = right.value else { + return None; + }; + let value = if subtract { + Value::Complex { + re: real - re, + im: -im, + } + } else { + Value::Complex { re: real + re, im } + }; + Some(Term::other(value)) +} + +/// Python equality for hashable literal values: numbers compare by value across `bool`, +/// `int`, `float`, and `complex`, so `1`, `1.0`, `True`, and `(1+0j)` are one key. +#[derive(Hash, PartialEq, Eq)] +enum KeyId { + None, + Int(BigInt), + Float(u64), + Complex(u64, u64), + Str(String), + Bytes(Vec), + Tuple(Vec), +} + +fn float_key(value: f64) -> KeyId { + if value.fract() == 0.0 + && let Some(integer) = BigInt::from_f64(value) + { + return KeyId::Int(integer); + } + KeyId::Float(value.to_bits()) +} + +fn key_id(value: &Value) -> Result { + Ok(match value { + Value::None => KeyId::None, + Value::Bool(value) => KeyId::Int(BigInt::from(u8::from(*value))), + Value::Int(value) => KeyId::Int(value.clone()), + Value::Float(value) => float_key(*value), + Value::Complex { re, im } if *im == 0.0 => float_key(*re), + Value::Complex { re, im } => KeyId::Complex((re + 0.0).to_bits(), (im + 0.0).to_bits()), + Value::Str(text) => KeyId::Str(text.clone()), + Value::Bytes(bytes) => KeyId::Bytes(bytes.clone()), + Value::Tuple(values) => KeyId::Tuple(values.iter().map(key_id).collect::>()?), + value @ (Value::List(_) | Value::Dict(_) | Value::Set(_)) => { + return Err(Error::Unhashable(value.type_name())); + } + }) +} + +/// Dict entries or set members in first-seen order: a repeated key keeps its first +/// position and, for dicts, takes the latest value. +#[derive(Default)] +struct UniqueValues { + keys: Vec, + values: Vec, + index: HashMap, +} + +impl UniqueValues { + fn insert(&mut self, key: Value, value: Option) -> Result<(), Error> { + match self.index.entry(key_id(&key)?) { + Entry::Occupied(slot) => { + if let Some(value) = value { + self.values[*slot.get()] = value; + } + } + Entry::Vacant(slot) => { + slot.insert(self.keys.len()); + self.keys.push(key); + self.values.extend(value); + } + } + Ok(()) + } + + fn into_entries(self) -> Vec<(Value, Value)> { + self.keys.into_iter().zip(self.values).collect() + } +} diff --git a/litellm-rust/crates/python-compat/src/pickle.rs b/litellm-rust/crates/python-compat/src/pickle.rs new file mode 100644 index 00000000000..bff7442dc6b --- /dev/null +++ b/litellm-rust/crates/python-compat/src/pickle.rs @@ -0,0 +1,187 @@ +//! `pickle.loads` and `pickle.dumps` for plain data, as diskcache stores LiteLLM values. +//! +//! Both directions go through `serde-pickle`'s serde interface rather than +//! `serde_pickle::Value`, because that value type keeps dicts in a `BTreeMap` and would +//! reorder keys. The serde interface keeps insertion order, at the cost of reporting +//! `tuple`, `set`, and `frozenset` as sequences: [`loads`] decodes all three as lists. +//! Python objects that need a class (`GLOBAL`/`REDUCE`) and recursive structures fail. + +use std::fmt; + +use serde::{ + Deserializer, Serialize, Serializer, + de::{self, DeserializeSeed, MapAccess, SeqAccess, Visitor}, + ser::{SerializeMap, SerializeSeq, SerializeTuple}, +}; +use serde_pickle::{DeOptions, SerOptions}; + +use crate::{Error, MAX_DEPTH, Value}; + +/// `pickle.loads(data)` for any protocol from 0 to 5. +pub fn loads(data: &[u8]) -> Result { + let mut deserializer = serde_pickle::Deserializer::new(data, DeOptions::new()); + let value = Seed { depth: 0 } + .deserialize(&mut deserializer) + .map_err(|error| Error::InvalidPickle(error.to_string()))?; + deserializer + .end() + .map_err(|error| Error::InvalidPickle(error.to_string()))?; + Ok(value) +} + +/// `pickle.dumps(value, protocol=3)`. Every Python 3 reads protocol 3, whatever its own +/// default. Sets and complex numbers are rejected rather than silently changing type. +pub fn dumps(value: &Value) -> Result, Error> { + check_picklable(value, 0)?; + serde_pickle::to_vec(&Pickled(value), SerOptions::new()) + .map_err(|error| Error::InvalidPickle(error.to_string())) +} + +fn check_picklable(value: &Value, depth: usize) -> Result<(), Error> { + if depth > MAX_DEPTH { + return Err(Error::TooDeep); + } + match value { + Value::Int(value) if i64::try_from(value).is_err() => Err(Error::IntegerOutOfRange), + value @ (Value::Set(_) | Value::Complex { .. }) => { + Err(Error::NotPicklable(value.type_name())) + } + Value::List(values) | Value::Tuple(values) => values + .iter() + .try_for_each(|value| check_picklable(value, depth + 1)), + Value::Dict(entries) => entries.iter().try_for_each(|(key, value)| { + check_picklable(key, depth + 1)?; + check_picklable(value, depth + 1) + }), + _ => Ok(()), + } +} + +struct Pickled<'a>(&'a Value); + +impl Serialize for Pickled<'_> { + fn serialize(&self, serializer: S) -> Result { + match self.0 { + Value::None => serializer.serialize_unit(), + Value::Bool(value) => serializer.serialize_bool(*value), + Value::Int(value) => { + let value = i64::try_from(value) + .map_err(|_| serde::ser::Error::custom("integer out of i64 range"))?; + serializer.serialize_i64(value) + } + Value::Float(value) => serializer.serialize_f64(*value), + Value::Str(text) => serializer.serialize_str(text), + Value::Bytes(bytes) => serializer.serialize_bytes(bytes), + Value::List(values) => { + let mut seq = serializer.serialize_seq(Some(values.len()))?; + for value in values { + seq.serialize_element(&Pickled(value))?; + } + seq.end() + } + Value::Tuple(values) => { + let mut tuple = serializer.serialize_tuple(values.len())?; + for value in values { + tuple.serialize_element(&Pickled(value))?; + } + tuple.end() + } + Value::Dict(entries) => { + let mut map = serializer.serialize_map(Some(entries.len()))?; + for (key, value) in entries { + map.serialize_entry(&Pickled(key), &Pickled(value))?; + } + map.end() + } + value @ (Value::Set(_) | Value::Complex { .. }) => Err(serde::ser::Error::custom( + format!("{} cannot be pickled as plain data", value.type_name()), + )), + } + } +} + +#[derive(Clone, Copy)] +struct Seed { + depth: usize, +} + +impl Seed { + fn child(self) -> Result { + if self.depth >= MAX_DEPTH { + return Err(E::custom(Error::TooDeep)); + } + Ok(Self { + depth: self.depth + 1, + }) + } +} + +impl<'de> DeserializeSeed<'de> for Seed { + type Value = Value; + + fn deserialize>(self, deserializer: D) -> Result { + deserializer.deserialize_any(self) + } +} + +impl<'de> Visitor<'de> for Seed { + type Value = Value; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a plain Python data value") + } + + fn visit_unit(self) -> Result { + Ok(Value::None) + } + + fn visit_bool(self, value: bool) -> Result { + Ok(Value::Bool(value)) + } + + fn visit_i64(self, value: i64) -> Result { + Ok(Value::Int(value.into())) + } + + fn visit_u64(self, value: u64) -> Result { + Ok(Value::Int(value.into())) + } + + fn visit_f64(self, value: f64) -> Result { + Ok(Value::Float(value)) + } + + fn visit_str(self, value: &str) -> Result { + Ok(Value::Str(value.to_owned())) + } + + fn visit_string(self, value: String) -> Result { + Ok(Value::Str(value)) + } + + fn visit_bytes(self, value: &[u8]) -> Result { + Ok(Value::Bytes(value.to_vec())) + } + + fn visit_byte_buf(self, value: Vec) -> Result { + Ok(Value::Bytes(value)) + } + + fn visit_seq>(self, mut seq: A) -> Result { + let child = self.child()?; + let mut values = Vec::with_capacity(seq.size_hint().unwrap_or(0).min(4096)); + while let Some(value) = seq.next_element_seed(child)? { + values.push(value); + } + Ok(Value::List(values)) + } + + fn visit_map>(self, mut map: A) -> Result { + let child = self.child()?; + let mut entries = Vec::with_capacity(map.size_hint().unwrap_or(0).min(4096)); + while let Some(key) = map.next_key_seed(child)? { + entries.push((key, map.next_value_seed(child)?)); + } + Ok(Value::Dict(entries)) + } +} diff --git a/litellm-rust/crates/python-compat/src/repr.rs b/litellm-rust/crates/python-compat/src/repr.rs new file mode 100644 index 00000000000..1cf20d87716 --- /dev/null +++ b/litellm-rust/crates/python-compat/src/repr.rs @@ -0,0 +1,237 @@ +//! `repr(value)` and `str(value)` byte for byte. +//! +//! LiteLLM hashes `str(value)` into cache keys and writes `str(dict)` into Redis, so these +//! strings are persisted identifiers rather than display text: every quote choice, escape, +//! and float digit must match CPython. + +use std::fmt::Write; + +use crate::Value; + +/// Generated by `scripts/generate_nonprintable.py`; see `generated/nonprintable.rs`. +mod nonprintable { + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/generated/nonprintable.rs" + )); +} + +/// Unicode version of the printable-character table, from the Python that generated it. +pub const UNICODE_VERSION: &str = nonprintable::UNICODE_VERSION; + +/// `repr(value)`. +pub fn repr(value: &Value) -> String { + let mut out = String::new(); + write_repr(&mut out, value); + out +} + +/// `str(value)`: a string's own contents, and `repr` for every other value. +pub fn to_str(value: &Value) -> String { + match value { + Value::Str(text) => text.clone(), + value => repr(value), + } +} + +/// `repr(float)`, the shortest round-trip form with CPython's exponent thresholds. +pub fn float_repr(value: f64) -> String { + format_float(value, true) +} + +fn write_repr(out: &mut String, value: &Value) { + match value { + Value::None => out.push_str("None"), + Value::Bool(true) => out.push_str("True"), + Value::Bool(false) => out.push_str("False"), + Value::Int(value) => { + let _ = write!(out, "{value}"); + } + Value::Float(value) => out.push_str(&float_repr(*value)), + Value::Complex { re, im } => write_complex(out, *re, *im), + Value::Str(text) => write_str(out, text), + Value::Bytes(bytes) => write_bytes(out, bytes), + Value::List(values) => write_sequence(out, '[', values, ']'), + Value::Tuple(values) if values.len() == 1 => { + out.push('('); + write_repr(out, &values[0]); + out.push_str(",)"); + } + Value::Tuple(values) => write_sequence(out, '(', values, ')'), + Value::Set(values) if values.is_empty() => out.push_str("set()"), + Value::Set(values) => write_sequence(out, '{', values, '}'), + Value::Dict(entries) => { + out.push('{'); + for (index, (key, value)) in entries.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + write_repr(out, key); + out.push_str(": "); + write_repr(out, value); + } + out.push('}'); + } + } +} + +fn write_sequence(out: &mut String, open: char, values: &[Value], close: char) { + out.push(open); + for (index, value) in values.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + write_repr(out, value); + } + out.push(close); +} + +/// `complex.__repr__`: a `+0.0` real part prints only the imaginary part, without parens. +fn write_complex(out: &mut String, re: f64, im: f64) { + if re == 0.0 && re.is_sign_positive() { + out.push_str(&format_float(im, false)); + out.push('j'); + return; + } + out.push('('); + out.push_str(&format_float(re, false)); + let im_text = format_float(im, false); + if !im_text.starts_with('-') { + out.push('+'); + } + out.push_str(&im_text); + out.push_str("j)"); +} + +/// `PyOS_double_to_string(value, 'r', 0, flags)`: scientific notation below 1e-4 and from +/// 1e16 up, with a signed exponent of at least two digits. `add_dot_0` is +/// `Py_DTSF_ADD_DOT_0`, which `float` sets and `complex` does not. +fn format_float(value: f64, add_dot_0: bool) -> String { + if value.is_nan() { + return "nan".to_owned(); + } + if value.is_infinite() { + return if value > 0.0 { "inf" } else { "-inf" }.to_owned(); + } + // Rust's `{:e}` prints the shortest round-trip digits, like CPython's 'r' mode. + let scientific = format!("{value:e}"); + let (mantissa, exponent) = scientific + .split_once('e') + .expect("`{:e}` always prints an exponent"); + let exponent: i32 = exponent.parse().expect("`{:e}` exponent is an integer"); + let (sign, mantissa) = match mantissa.strip_prefix('-') { + Some(mantissa) => ("-", mantissa), + None => ("", mantissa), + }; + let digits: String = mantissa.chars().filter(|ch| *ch != '.').collect(); + + let mut out = String::from(sign); + if !(-4..16).contains(&exponent) { + out.push_str(&digits[..1]); + if digits.len() > 1 { + out.push('.'); + out.push_str(&digits[1..]); + } + let _ = write!( + out, + "e{}{:02}", + if exponent < 0 { '-' } else { '+' }, + exponent.unsigned_abs() + ); + } else if exponent < 0 { + out.push_str("0."); + out.extend(std::iter::repeat_n('0', (-exponent - 1) as usize)); + out.push_str(&digits); + } else { + let integer_digits = exponent as usize + 1; + if digits.len() > integer_digits { + out.push_str(&digits[..integer_digits]); + out.push('.'); + out.push_str(&digits[integer_digits..]); + } else { + out.push_str(&digits); + out.extend(std::iter::repeat_n('0', integer_digits - digits.len())); + if add_dot_0 { + out.push_str(".0"); + } + } + } + out +} + +/// `unicode_repr`: single quotes unless the text has a `'` and no `"`. Printable non-ASCII +/// stays literal; everything `str.isprintable()` rejects is escaped. +fn write_str(out: &mut String, text: &str) { + let quote = if text.contains('\'') && !text.contains('"') { + '"' + } else { + '\'' + }; + out.push(quote); + for ch in text.chars() { + match ch { + '\\' => out.push_str("\\\\"), + '\t' => out.push_str("\\t"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + ch if ch == quote => { + out.push('\\'); + out.push(ch); + } + ch if is_printable(ch) => out.push(ch), + ch => { + let code = ch as u32; + let _ = match code { + 0..=0xff => write!(out, "\\x{code:02x}"), + 0x100..=0xffff => write!(out, "\\u{code:04x}"), + _ => write!(out, "\\U{code:08x}"), + }; + } + } + } + out.push(quote); +} + +/// `bytes.__repr__`: the same quote rule as `str`, with every byte outside printable ASCII +/// escaped as `\xhh`. +fn write_bytes(out: &mut String, bytes: &[u8]) { + let quote = if bytes.contains(&b'\'') && !bytes.contains(&b'"') { + b'"' + } else { + b'\'' + }; + out.push('b'); + out.push(quote as char); + for &byte in bytes { + match byte { + b'\\' => out.push_str("\\\\"), + b'\t' => out.push_str("\\t"), + b'\n' => out.push_str("\\n"), + b'\r' => out.push_str("\\r"), + byte if byte == quote => { + out.push('\\'); + out.push(byte as char); + } + 0x20..=0x7e => out.push(byte as char), + byte => { + let _ = write!(out, "\\x{byte:02x}"); + } + } + } + out.push(quote as char); +} + +fn is_printable(ch: char) -> bool { + let code = ch as u32; + nonprintable::NONPRINTABLE + .binary_search_by(|&(low, high)| { + if high < code { + std::cmp::Ordering::Less + } else if low > code { + std::cmp::Ordering::Greater + } else { + std::cmp::Ordering::Equal + } + }) + .is_err() +} diff --git a/litellm-rust/crates/python-compat/src/truthy.rs b/litellm-rust/crates/python-compat/src/truthy.rs new file mode 100644 index 00000000000..af747202572 --- /dev/null +++ b/litellm-rust/crates/python-compat/src/truthy.rs @@ -0,0 +1,18 @@ +use num_bigint::Sign; + +use crate::Value; + +/// `bool(value)` for plain data: `None`, `False`, zero, and empty containers are false. +pub fn truthy(value: &Value) -> bool { + match value { + Value::None => false, + Value::Bool(value) => *value, + Value::Int(value) => value.sign() != Sign::NoSign, + Value::Float(value) => *value != 0.0, + Value::Complex { re, im } => *re != 0.0 || *im != 0.0, + Value::Str(value) => !value.is_empty(), + Value::Bytes(value) => !value.is_empty(), + Value::Tuple(values) | Value::List(values) | Value::Set(values) => !values.is_empty(), + Value::Dict(entries) => !entries.is_empty(), + } +} diff --git a/litellm-rust/crates/python-compat/src/value.rs b/litellm-rust/crates/python-compat/src/value.rs new file mode 100644 index 00000000000..b1397190638 --- /dev/null +++ b/litellm-rust/crates/python-compat/src/value.rs @@ -0,0 +1,53 @@ +use num_bigint::BigInt; + +/// A Python value built only from literals: what `ast.literal_eval` can return. +#[derive(Clone, Debug, PartialEq)] +pub enum Value { + None, + Bool(bool), + Int(BigInt), + Float(f64), + Complex { + re: f64, + im: f64, + }, + Str(String), + Bytes(Vec), + Tuple(Vec), + List(Vec), + /// Insertion-ordered, with Python's key equality already applied. + Dict(Vec<(Value, Value)>), + /// Literal order, with Python's member equality already applied. + Set(Vec), +} + +impl Value { + /// Python's type name, as it appears in `TypeError` messages. + pub fn type_name(&self) -> &'static str { + match self { + Value::None => "NoneType", + Value::Bool(_) => "bool", + Value::Int(_) => "int", + Value::Float(_) => "float", + Value::Complex { .. } => "complex", + Value::Str(_) => "str", + Value::Bytes(_) => "bytes", + Value::Tuple(_) => "tuple", + Value::List(_) => "list", + Value::Dict(_) => "dict", + Value::Set(_) => "set", + } + } +} + +impl From for Value { + fn from(value: i64) -> Self { + Value::Int(value.into()) + } +} + +impl From<&str> for Value { + fn from(value: &str) -> Self { + Value::Str(value.to_owned()) + } +} diff --git a/litellm-rust/crates/python-compat/tests/fixtures.rs b/litellm-rust/crates/python-compat/tests/fixtures.rs new file mode 100644 index 00000000000..9c202fdc5f1 --- /dev/null +++ b/litellm-rust/crates/python-compat/tests/fixtures.rs @@ -0,0 +1,343 @@ +//! Replays `generated/values.json`, which CPython wrote with `scripts/generate_fixtures.py`. + +use std::{collections::BTreeMap, fs::File, io::Write}; + +use litellm_python_compat::{ + Value, json, + literal::literal_eval, + pickle, + repr::{repr, to_str}, + truthy::truthy, +}; +use rstest::{fixture, rstest}; +use serde::Deserialize; + +#[derive(Deserialize)] +struct Fixtures { + rows: Vec, + sources: Vec, +} + +/// `ast.literal_eval(source)`: the `repr` of its result, or the exception class it raised. +#[derive(Deserialize)] +struct Source { + name: String, + source: String, + repr: Option, + error: Option, +} + +#[derive(Deserialize)] +struct Row { + name: String, + source: String, + literal: bool, + plain: bool, + repr: String, + str: String, + truthy: bool, + json: Option, + json_error: Option, + pickle: Option>, + view: Option, +} + +/// Parsed once for the whole test binary. +#[fixture] +#[once] +fn fixtures() -> Fixtures { + serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/generated/values.json" + ))) + .expect("values.json matches the fixture schema") +} + +/// Accepted differences from CPython: `(fixture source, check prefix, reason)`. Each entry +/// must still differ, so a dependency fix that removes one fails the test until it is deleted. +const KNOWN: &[(&str, &str, &str)] = &[ + ( + "...", + "literal_eval source", + "`Ellipsis` is not part of the data model", + ), + ( + r"'\ud800'", + "literal_eval source", + "a Rust `String` cannot hold a lone surrogate", + ), + ( + r"'\N{BULLET}'", + "literal_eval source", + "`\\N{NAME}` needs the Unicode name table", + ), + ( + "nested_150", + "literal_eval", + "deeper than MAX_DEPTH: rejected for stack safety, where CPython still parses it", + ), + ( + "nested_150", + "pickle.loads", + "deeper than MAX_DEPTH: rejected for stack safety, where CPython has no limit", + ), + ( + "2**64", + "pickle.loads", + "serde-pickle's serde interface stops at i64", + ), + ( + "-(2**70)", + "pickle.loads", + "serde-pickle's serde interface stops at i64", + ), + ( + "2**64", + "pickle.dumps", + "serde-pickle's serde interface stops at i64", + ), + ( + "-(2**70)", + "pickle.dumps", + "serde-pickle's serde interface stops at i64", + ), + ( + "2**64", + "to_json", + "serde_json has no exact form for integers beyond u64", + ), + ( + "-(2**70)", + "to_json", + "serde_json has no exact form for integers beyond i64", + ), + ( + "b''", + "pickle.loads protocol 0", + "protocols 0-2 pickle `b''` as a `bytes()` call", + ), + ( + "b''", + "pickle.loads protocol 1", + "protocols 0-2 pickle `b''` as a `bytes()` call", + ), + ( + "b''", + "pickle.loads protocol 2", + "protocols 0-2 pickle `b''` as a `bytes()` call", + ), +]; + +/// Collects every mismatch so one run reports the whole divergence set. +#[derive(Default)] +struct Mismatches { + unexpected: Vec, + known_seen: Vec, + known_scope: Vec, +} + +impl Mismatches { + fn known(source: &str, what: &str) -> Option { + KNOWN + .iter() + .position(|(known, prefix, _)| *known == source && what.starts_with(prefix)) + } + + fn check(&mut self, row: &Row, what: &str, expected: &str, actual: &str) { + self.check_source(&row.name, what, expected, actual); + } + + /// `name` identifies the fixture row in reports and in [`KNOWN`]. + fn check_source(&mut self, name: &str, what: &str, expected: &str, actual: &str) { + let known = Self::known(name, what); + if let Some(index) = known { + self.known_scope.push(index); + } + if expected == actual { + return; + } + match known { + Some(index) => self.known_seen.push(index), + None => self.unexpected.push(format!( + "{name:?} [{what}]\n python: {expected}\n rust: {actual}" + )), + } + } + + fn finish(self) { + let resolved: Vec<_> = self + .known_scope + .iter() + .filter(|index| !self.known_seen.contains(index)) + .map(|&index| format!("{} [{}]", KNOWN[index].0, KNOWN[index].1)) + .collect(); + assert!( + self.unexpected.is_empty() && resolved.is_empty(), + "{} mismatches with CPython:\n{}\nknown divergences that now match (delete them \ + from KNOWN): {resolved:?}", + self.unexpected.len(), + self.unexpected.join("\n"), + ); + } +} + +fn check_value(mismatches: &mut Mismatches, row: &Row, value: &Value) { + mismatches.check(row, "repr", &row.repr, &repr(value)); + mismatches.check(row, "str", &row.str, &to_str(value)); + mismatches.check( + row, + "bool", + &row.truthy.to_string(), + &truthy(value).to_string(), + ); + let expected = row.json.clone().or_else(|| { + row.json_error + .clone() + .map(|error| format!("error: {error}")) + }); + let actual = match json::dumps(value) { + Ok(text) => text, + Err(error) => format!("error: {error}"), + }; + mismatches.check( + row, + "json.dumps", + expected.as_deref().unwrap_or(""), + &actual, + ); +} + +#[rstest] +fn literal_rows_match_python_repr_str_bool_and_json(fixtures: &Fixtures) { + let mut mismatches = Mismatches::default(); + for row in fixtures.rows.iter().filter(|row| row.literal) { + match literal_eval(&row.repr) { + Ok(value) => check_value(&mut mismatches, row, &value), + Err(error) => mismatches.check(row, "literal_eval", &row.repr, &error.to_string()), + } + } + mismatches.finish(); +} + +/// Errors compare by outcome only: CPython's exception class is not part of the contract. +#[rstest] +fn literal_eval_matches_python_on_source_texts(fixtures: &Fixtures) { + let mut mismatches = Mismatches::default(); + for case in &fixtures.sources { + let expected = match (&case.repr, &case.error) { + (Some(repr), None) => repr.clone(), + (None, Some(_)) => "an error".to_owned(), + _ => panic!("{:?}: a source records a repr or an error", case.source), + }; + let actual = match literal_eval(&case.source) { + Ok(value) => repr(&value), + Err(_) => "an error".to_owned(), + }; + mismatches.check_source(&case.name, "literal_eval source", &expected, &actual); + } + mismatches.finish(); +} + +#[rstest] +fn pickle_loads_matches_python_at_every_protocol(fixtures: &Fixtures) { + let mut mismatches = Mismatches::default(); + for row in &fixtures.rows { + let Some(pickles) = &row.pickle else { continue }; + for (protocol, data) in pickles { + let data = hex::decode(data).expect("fixture pickle is hex"); + let what = format!("pickle.loads protocol {protocol}"); + match (pickle::loads(&data), row.plain) { + (Ok(value), true) => { + let view = row.view.as_deref().expect("picklable rows have a view"); + mismatches.check(row, &what, view, &repr(&value)); + } + (Err(error), true) => { + mismatches.check(row, &what, "a value", &format!("error: {error}")) + } + (Ok(value), false) => { + mismatches.check(row, &what, "a class-reference error", &repr(&value)) + } + (Err(pickle_error), false) => assert!( + matches!(pickle_error, litellm_python_compat::Error::InvalidPickle(_)), + "{}: {pickle_error}", + row.source + ), + } + } + } + mismatches.finish(); +} + +/// Non-finite floats have no literal form; pickle is how Rust receives them. +#[rstest] +fn values_reached_only_through_pickle_match_python(fixtures: &Fixtures) { + let mut mismatches = Mismatches::default(); + for row in fixtures + .rows + .iter() + .filter(|row| !row.literal && row.plain && row.view.as_deref() == Some(&row.repr)) + { + let data = hex::decode(&row.pickle.as_ref().expect("plain rows pickle")["5"]) + .expect("fixture pickle is hex"); + let value = pickle::loads(&data).expect("plain pickle decodes"); + check_value(&mut mismatches, row, &value); + } + mismatches.finish(); +} + +/// Byte equality with CPython is not the contract: CPython adds memo opcodes and picks the +/// smallest integer opcode. `scripts/verify_rust_pickles.py` checks that CPython reads these +/// back; set `PYTHON_COMPAT_RUST_PICKLES` to a file path to export them. +#[rstest] +fn pickle_dumps_round_trips_every_plain_literal(fixtures: &Fixtures) { + // Truncate up front: the verifier must read this run's rows and nothing else. + let mut export = std::env::var_os("PYTHON_COMPAT_RUST_PICKLES") + .map(|path| File::create(path).expect("the export path is writable")); + let mut mismatches = Mismatches::default(); + for row in fixtures + .rows + .iter() + .filter(|row| row.literal && row.plain && row.view.as_deref() == Some(&row.repr)) + { + // Rows past `MAX_DEPTH` are covered by the literal test's own KNOWN entry. + let Ok(value) = literal_eval(&row.repr) else { + continue; + }; + let data = match pickle::dumps(&value) { + Ok(data) => data, + Err(error) => { + mismatches.check(row, "pickle.dumps", "a pickle", &format!("error: {error}")); + continue; + } + }; + let decoded = pickle::loads(&data).expect("rust pickle decodes"); + mismatches.check(row, "pickle round trip", &row.repr, &repr(&decoded)); + if let Some(file) = &mut export { + writeln!(file, "{}\t{}", hex::encode(&data), repr(&value)) + .expect("the export file is writable"); + } + } + mismatches.finish(); +} + +#[rstest] +fn to_json_matches_python_json_round_trip(fixtures: &Fixtures) { + let mut mismatches = Mismatches::default(); + for row in fixtures.rows.iter().filter(|row| row.literal) { + let Some(expected) = &row.json else { continue }; + let Ok(value) = literal_eval(&row.repr) else { + continue; + }; + let expected: serde_json::Value = + serde_json::from_str(expected).expect("python json.dumps output parses"); + match json::to_json(&value) { + Ok(actual) => { + mismatches.check(row, "to_json", &expected.to_string(), &actual.to_string()) + } + Err(error) => { + mismatches.check(row, "to_json", &expected.to_string(), &error.to_string()) + } + } + } + mismatches.finish(); +} diff --git a/litellm-rust/crates/python-compat/tests/limits.rs b/litellm-rust/crates/python-compat/tests/limits.rs new file mode 100644 index 00000000000..c4d931034c4 --- /dev/null +++ b/litellm-rust/crates/python-compat/tests/limits.rs @@ -0,0 +1,122 @@ +use std::time::{Duration, Instant}; + +use litellm_python_compat::{Error, MAX_DEPTH, Value, json, literal::literal_eval, pickle}; +use rstest::{fixture, rstest}; + +/// The bracket pair of one container shape, as `(open, close)`. +#[fixture] +fn shapes() -> [(&'static str, &'static str); 3] { + [("[", "]"), ("{'a': ", "}"), ("(", ",)")] +} + +fn nested_text(open: &str, close: &str, depth: usize) -> String { + format!("{}1{}", open.repeat(depth), close.repeat(depth)) +} + +fn nested_list(depth: usize) -> Value { + (0..depth).fold(Value::from(1), |value, _| Value::List(vec![value])) +} + +/// A protocol 3 pickle of `depth` nested lists around `1`: `EMPTY_LIST` per level, then +/// `BININT1 1`, then `APPEND` per level. Written by hand because `dumps` refuses the depth. +fn nested_list_pickle(depth: usize) -> Vec { + let mut data = vec![0x80, 3]; + data.extend(std::iter::repeat_n(b']', depth)); + data.extend([b'K', 1]); + data.extend(std::iter::repeat_n(b'a', depth)); + data.push(b'.'); + data +} + +#[rstest] +fn literal_eval_accepts_the_limit_and_rejects_past_it(shapes: [(&'static str, &'static str); 3]) { + for (open, close) in shapes { + assert!(literal_eval(&nested_text(open, close, MAX_DEPTH)).is_ok()); + assert!(matches!( + literal_eval(&nested_text(open, close, MAX_DEPTH + 1)), + Err(Error::TooDeep) + )); + } +} + +/// A backtracking parser (the `py_literal` grammar this replaced) doubles per nested level +/// and takes minutes here; the bound is loose enough to survive a slow debug build. +#[rstest] +fn literal_eval_stays_linear_in_depth(shapes: [(&'static str, &'static str); 3]) { + for (open, close) in shapes { + let text = nested_text(open, close, MAX_DEPTH); + let start = Instant::now(); + assert!(literal_eval(&text).is_ok()); + let elapsed = start.elapsed(); + assert!( + elapsed < Duration::from_millis(50), + "{open} nested {MAX_DEPTH} deep took {elapsed:?}" + ); + } +} + +#[rstest] +fn literal_eval_ignores_brackets_inside_strings() { + let text = format!("'{}'", "[".repeat(MAX_DEPTH + 1)); + assert!(matches!(literal_eval(&text), Ok(Value::Str(_)))); +} + +#[rstest] +fn pickle_nesting_is_bounded_in_both_directions() { + assert_eq!( + pickle::loads(&nested_list_pickle(MAX_DEPTH)).unwrap(), + nested_list(MAX_DEPTH) + ); + assert!(matches!( + pickle::loads(&nested_list_pickle(MAX_DEPTH + 1)), + Err(Error::InvalidPickle(_)) + )); + assert!(pickle::dumps(&nested_list(MAX_DEPTH)).is_ok()); + assert!(matches!( + pickle::dumps(&nested_list(MAX_DEPTH + 2)), + Err(Error::TooDeep) + )); +} + +#[rstest] +#[case("{1, 2}", "set")] +#[case("1+2j", "complex")] +fn pickle_dumps_refuses_types_it_would_change(#[case] source: &str, #[case] type_name: &str) { + let value = literal_eval(source).expect("source is a literal"); + assert!(matches!(pickle::dumps(&value), Err(Error::NotPicklable(name)) if name == type_name)); +} + +#[rstest] +#[case(b"\x80\x02c__builtin__\ncomplex\nq\x00.".to_vec(), "a class reference")] +#[case({ let mut data = pickle::dumps(&Value::from(1)).unwrap(); data.push(b'.'); data }, "trailing data")] +fn pickle_loads_rejects(#[case] data: Vec, #[case] what: &str) { + assert!( + matches!(pickle::loads(&data), Err(Error::InvalidPickle(_))), + "{what} must not decode" + ); +} + +#[rstest] +#[case(Value::Bytes(b"x".to_vec()), "Object of type bytes is not JSON serializable")] +#[case(Value::Set(vec![Value::from(1)]), "Object of type set is not JSON serializable")] +#[case(Value::Complex { re: 1.0, im: 2.0 }, "Object of type complex is not JSON serializable")] +#[case( + Value::Float(f64::NAN), + "Out of range float values are not JSON compliant" +)] +fn to_json_reports_what_python_json_dumps_would_reject( + #[case] value: Value, + #[case] message: &str, +) { + let error = json::to_json(&value).expect_err("value has no serde_json form"); + assert_eq!(error.to_string(), message); +} + +/// `json.dumps` writes the non-finite floats that `to_json` cannot represent. +#[rstest] +#[case(f64::NAN, "NaN")] +#[case(f64::INFINITY, "Infinity")] +#[case(f64::NEG_INFINITY, "-Infinity")] +fn json_dumps_writes_non_finite_floats(#[case] value: f64, #[case] text: &str) { + assert_eq!(json::dumps(&Value::Float(value)).unwrap(), text); +} diff --git a/litellm-rust/crates/secrets-aws/src/error.rs b/litellm-rust/crates/secrets-aws/src/error.rs index 23595397a13..cc9e6b69786 100644 --- a/litellm-rust/crates/secrets-aws/src/error.rs +++ b/litellm-rust/crates/secrets-aws/src/error.rs @@ -6,6 +6,10 @@ pub enum Error { Auth(#[from] #[redact] litellm_auth_aws::Error), #[error("AWS region is not configured")] MissingRegion, + #[error("AWS Secrets Manager received a non-AWS operation context")] + InvalidOperationContext, + #[error("AWS Secrets Manager was constructed without context-aware configuration")] + OperationContextUnavailable, #[error("KMS response has no plaintext")] MissingPlaintext, #[error("AWS request timed out")] diff --git a/litellm-rust/crates/secrets-aws/src/secret_manager.rs b/litellm-rust/crates/secrets-aws/src/secret_manager.rs index 493cb1d2e8f..76c61534861 100644 --- a/litellm-rust/crates/secrets-aws/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-aws/src/secret_manager.rs @@ -16,7 +16,8 @@ use litellm_auth_aws::constants::{ }; use litellm_core_utils::settings::Lookup; use litellm_secrets_types::{ - BaseSecretManager, KeyManagementSettings, Secret, SecretValue, async_rotate_secret, + AwsOperationContext, BaseSecretManager, KeyManagementSettings, Secret, SecretOperationContext, + SecretValue, SecretWriteContext, async_rotate_secret, }; use serde_json::Value; @@ -25,9 +26,17 @@ use crate::{Error, auth}; #[derive(Clone)] pub struct AwsSecretsManagerV2 { client: Client, + context_client_factory: Option>, write_settings: AwsSecretWriteSettings, } +#[derive(Clone)] +struct ContextClientFactory { + settings: KeyManagementSettings, + environment: Arc, + endpoint_url: Option, +} + #[derive(Clone, Debug, Default)] pub struct AwsSecretWriteSettings { pub kms_key_id: Option, @@ -55,6 +64,19 @@ impl AwsSecretsManagerV2 { pub fn new(client: Client, write_settings: AwsSecretWriteSettings) -> Self { Self { client, + context_client_factory: None, + write_settings, + } + } + + fn with_context_client_factory( + client: Client, + write_settings: AwsSecretWriteSettings, + context_client_factory: ContextClientFactory, + ) -> Self { + Self { + client, + context_client_factory: Some(Box::new(context_client_factory)), write_settings, } } @@ -67,19 +89,18 @@ impl AwsSecretsManagerV2 { if use_aws_secret_manager != Some(true) { return Ok(None); } - let builder = aws_sdk_secretsmanager::Config::builder() - .behavior_version(BehaviorVersion::latest()) - .region(Region::new(auth::region(&settings, environment.as_ref())?)) - .credentials_provider(auth::Credentials::new(&settings, environment.clone())); - let config = match environment.get(AWS_BEDROCK_RUNTIME_ENDPOINT) { - Some(url) => builder - .endpoint_url(url.replace("bedrock-runtime", "secretsmanager")) - .build(), - None => builder.build(), + let context_client_factory = ContextClientFactory { + settings: settings.clone(), + environment: environment.clone(), + endpoint_url: environment + .get(AWS_BEDROCK_RUNTIME_ENDPOINT) + .map(|url| url.replace("bedrock-runtime", "secretsmanager")), }; - Ok(Some(Self::new( - Client::from_conf(config), + let client = context_client_factory.client(&AwsOperationContext::default())?; + Ok(Some(Self::with_context_client_factory( + client, (&settings).into(), + context_client_factory, ))) } @@ -118,7 +139,14 @@ impl AwsSecretsManagerV2 { } pub async fn async_read_secret(&self, name: &str) -> Result, Error> { - match self.client.get_secret_value().secret_id(name).send().await { + Self::async_read_secret_with_client(&self.client, name).await + } + + async fn async_read_secret_with_client( + client: &Client, + name: &str, + ) -> Result, Error> { + match client.get_secret_value().secret_id(name).send().await { Ok(response) => response .secret_string .map(SecretValue::new) @@ -149,8 +177,19 @@ impl AwsSecretsManagerV2 { value: &SecretValue, description: Option<&str>, ) -> Result { - let response = self - .client + self.async_write_secret_with_client_and_tags(&self.client, name, value, description, None) + .await + } + + async fn async_write_secret_with_client_and_tags( + &self, + client: &Client, + name: &str, + value: &SecretValue, + description: Option<&str>, + tags: Option<&BTreeMap>, + ) -> Result { + let response = client .create_secret() .name(name) .secret_string(value.expose()) @@ -161,7 +200,7 @@ impl AwsSecretsManagerV2 { .clone() .filter(|v| !v.is_empty()), ) - .set_tags(self.write_settings.tags.as_ref().map(|tags| { + .set_tags(tags.or(self.write_settings.tags.as_ref()).map(|tags| { tags.iter() .map(|(key, value)| Tag::builder().key(key).value(value).build()) .collect() @@ -171,7 +210,10 @@ impl AwsSecretsManagerV2 { .map_err(|error| Error::Create(Box::new(error)))?; if let Some(regions) = &self.write_settings.replica_regions && !regions.is_empty() - && self.async_replicate_secret(name, regions).await.is_err() + && self + .async_replicate_secret_with_client(client, name, regions) + .await + .is_err() { tracing::warn!("secret created but replication failed"); } @@ -182,11 +224,21 @@ impl AwsSecretsManagerV2 { &self, name: &str, regions: &[String], + ) -> Result, Error> { + self.async_replicate_secret_with_client(&self.client, name, regions) + .await + } + + async fn async_replicate_secret_with_client( + &self, + client: &Client, + name: &str, + regions: &[String], ) -> Result, Error> { if regions.is_empty() { return Ok(None); } - self.client + client .replicate_secret_to_regions() .secret_id(name) .set_add_replica_regions(Some( @@ -206,7 +258,17 @@ impl AwsSecretsManagerV2 { name: &str, value: &SecretValue, ) -> Result { - self.client + self.async_put_secret_value_with_client(&self.client, name, value) + .await + } + + async fn async_put_secret_value_with_client( + &self, + client: &Client, + name: &str, + value: &SecretValue, + ) -> Result { + client .put_secret_value() .secret_id(name) .secret_string(value.expose()) @@ -218,12 +280,22 @@ impl AwsSecretsManagerV2 { pub async fn async_delete_secret( &self, name: &str, - recovery_window_in_days: i64, + recovery_window_in_days: Option, ) -> Result { - self.client + self.async_delete_secret_with_client(&self.client, name, recovery_window_in_days) + .await + } + + async fn async_delete_secret_with_client( + &self, + client: &Client, + name: &str, + recovery_window_in_days: Option, + ) -> Result { + client .delete_secret() .secret_id(name) - .recovery_window_in_days(recovery_window_in_days) + .set_recovery_window_in_days(recovery_window_in_days.map(i64::from)) .send() .await .map_err(|error| Error::Delete(Box::new(error))) @@ -234,17 +306,105 @@ impl AwsSecretsManagerV2 { current_name: &str, new_name: &str, value: &SecretValue, + ) -> Result { + self.async_rotate_secret_with_context( + current_name, + new_name, + value, + &SecretOperationContext::default(), + ) + .await + } + + pub async fn async_rotate_secret_with_context( + &self, + current_name: &str, + new_name: &str, + value: &SecretValue, + context: &SecretOperationContext, ) -> Result { if current_name == new_name { + let client = self.client_for_context(context)?; return self - .async_put_secret_value(current_name, value) + .async_put_secret_value_with_client(&client, current_name, value) .await .map(RotationResponse::Updated); } - async_rotate_secret(self, current_name, new_name, value) + async_rotate_secret(self, current_name, new_name, value, context) .await .map(RotationResponse::Created) } + + fn client_for_context(&self, context: &SecretOperationContext) -> Result { + match context { + SecretOperationContext::Default => Ok(self.client.clone()), + SecretOperationContext::Aws(context) if context == &AwsOperationContext::default() => { + Ok(self.client.clone()) + } + SecretOperationContext::Aws(context) => self + .context_client_factory + .as_ref() + .ok_or(Error::OperationContextUnavailable)? + .client(context), + _ => Err(Error::InvalidOperationContext), + } + } +} + +impl ContextClientFactory { + fn client(&self, context: &AwsOperationContext) -> Result { + let settings = KeyManagementSettings { + aws_region_name: context + .region_name + .clone() + .or_else(|| self.settings.aws_region_name.clone()), + aws_role_name: context + .role_name + .clone() + .or_else(|| self.settings.aws_role_name.clone()), + aws_session_name: context + .session_name + .clone() + .or_else(|| self.settings.aws_session_name.clone()), + aws_external_id: context + .external_id + .clone() + .or_else(|| self.settings.aws_external_id.clone()), + aws_profile_name: context + .profile_name + .clone() + .or_else(|| self.settings.aws_profile_name.clone()), + aws_web_identity_token: context + .web_identity_token + .clone() + .or_else(|| self.settings.aws_web_identity_token.clone()), + aws_sts_endpoint: context + .sts_endpoint + .clone() + .or_else(|| self.settings.aws_sts_endpoint.clone()), + ..self.settings.clone() + }; + let builder = aws_sdk_secretsmanager::Config::builder() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new(auth::region( + &settings, + self.environment.as_ref(), + )?)) + .credentials_provider(auth::Credentials::new(&settings, self.environment.clone())); + let builder = match context.timeout { + Some(timeout) => builder.timeout_config( + aws_sdk_secretsmanager::config::timeout::TimeoutConfig::builder() + .operation_timeout(timeout) + .build(), + ), + None => builder, + }; + let config = match &self.endpoint_url { + Some(endpoint_url) => builder.endpoint_url(endpoint_url.clone()).build(), + None => builder.build(), + }; + Ok(Client::from_conf(config)) + } } impl BaseSecretManager for AwsSecretsManagerV2 { @@ -252,25 +412,40 @@ impl BaseSecretManager for AwsSecretsManagerV2 { type WriteResponse = CreateSecretOutput; type DeleteResponse = DeleteSecretOutput; - async fn async_read_secret(&self, name: &str) -> Result, Error> { - self.async_read_secret(name).await + async fn async_read_secret( + &self, + name: &str, + context: &SecretOperationContext, + ) -> Result, Error> { + let client = self.client_for_context(context)?; + Self::async_read_secret_with_client(&client, name).await } async fn async_write_secret( &self, name: &str, value: &SecretValue, - description: Option<&str>, + context: &SecretWriteContext, ) -> Result { - self.async_write_secret(name, value, description).await + let client = self.client_for_context(&context.operation)?; + self.async_write_secret_with_client_and_tags( + &client, + name, + value, + context.description.as_deref(), + (!context.tags.is_empty()).then_some(&context.tags), + ) + .await } async fn async_delete_secret( &self, name: &str, - recovery_window_in_days: i64, + recovery_window_in_days: Option, + context: &SecretOperationContext, ) -> Result { - self.async_delete_secret(name, recovery_window_in_days) + let client = self.client_for_context(context)?; + self.async_delete_secret_with_client(&client, name, recovery_window_in_days) .await } } diff --git a/litellm-rust/crates/secrets-aws/tests/kms.rs b/litellm-rust/crates/secrets-aws/tests/kms.rs index 39a50297551..687cca104e0 100644 --- a/litellm-rust/crates/secrets-aws/tests/kms.rs +++ b/litellm-rust/crates/secrets-aws/tests/kms.rs @@ -3,12 +3,15 @@ use aws_sdk_kms::{ config::{BehaviorVersion, Credentials, Region, retry::RetryConfig}, }; use base64::{Engine, engine::general_purpose::STANDARD}; -use litellm_secrets_aws::AwsKms; +use litellm_secrets_aws::{AwsKms, Error, load_aws_kms}; +use litellm_secrets_types::KeyManagementSettings; +use rstest::rstest; use wiremock::{ Mock, MockServer, ResponseTemplate, matchers::{body_json, header}, }; +#[rstest] #[tokio::test] async fn kms_decrypt_calls_the_sdk_without_applying_lookup_policy() { let server = MockServer::start().await; @@ -40,20 +43,56 @@ async fn kms_decrypt_calls_the_sdk_without_applying_lookup_policy() { ); } -#[test] -fn disabled_kms_loader_does_not_require_environment_configuration() { - use litellm_secrets_aws::load_aws_kms; - use litellm_secrets_types::KeyManagementSettings; +#[rstest] +#[case::unset(None)] +#[case::disabled(Some(false))] +fn disabled_kms_loader_does_not_require_environment_configuration(#[case] enabled: Option) { use std::sync::Arc; - for enabled in [None, Some(false)] { - assert!( - load_aws_kms( - enabled, - &KeyManagementSettings::default(), - Arc::new(|_: &str| None) - ) - .unwrap() - .is_none() - ); - } + assert!( + load_aws_kms( + enabled, + &KeyManagementSettings::default(), + Arc::new(|_: &str| None) + ) + .unwrap() + .is_none() + ); +} + +#[rstest] +#[case::settings(Some("configured-region"), None)] +#[case::environment(None, Some("environment-region"))] +fn enabled_kms_loader_accepts_either_region_source( + #[case] configured_region: Option<&'static str>, + #[case] environment_region: Option<&'static str>, +) { + use std::sync::Arc; + let settings = KeyManagementSettings { + aws_region_name: configured_region.map(str::to_owned), + ..KeyManagementSettings::default() + }; + let environment = Arc::new(move |name: &str| { + (name == "AWS_REGION_NAME") + .then(|| environment_region.map(str::to_owned)) + .flatten() + }); + + assert!( + load_aws_kms(Some(true), &settings, environment) + .unwrap() + .is_some() + ); +} + +#[rstest] +fn enabled_kms_loader_rejects_missing_region() { + use std::sync::Arc; + assert!(matches!( + load_aws_kms( + Some(true), + &KeyManagementSettings::default(), + Arc::new(|_: &str| None), + ), + Err(Error::MissingRegion) + )); } diff --git a/litellm-rust/crates/secrets-aws/tests/secret_manager.rs b/litellm-rust/crates/secrets-aws/tests/secret_manager.rs index a410767cb5a..7dbc8b61374 100644 --- a/litellm-rust/crates/secrets-aws/tests/secret_manager.rs +++ b/litellm-rust/crates/secrets-aws/tests/secret_manager.rs @@ -1,11 +1,21 @@ -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::{ + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, +}; use aws_sdk_secretsmanager::{ Client, config::{BehaviorVersion, Credentials, Region, retry::RetryConfig}, }; use litellm_secrets_aws::{AwsSecretsManagerV2, Error, RotationResponse}; -use litellm_secrets_types::{KeyManagementSettings, SecretValue}; +use litellm_secrets_types::{ + AwsOperationContext, BaseSecretManager, KeyManagementSettings, SecretOperationContext, + SecretValue, SecretWriteContext, +}; +use rstest::{fixture, rstest}; use serde_json::json; use wiremock::{ Mock, MockServer, ResponseTemplate, @@ -25,12 +35,39 @@ fn manager(server: &MockServer, settings: KeyManagementSettings) -> AwsSecretsMa AwsSecretsManagerV2::new(client, (&settings).into()) } -#[rstest::rstest] +fn loaded_manager(server: &MockServer) -> AwsSecretsManagerV2 { + let endpoint_url = server.uri(); + let environment: Arc = + Arc::new(move |name: &str| match name { + "AWS_BEDROCK_RUNTIME_ENDPOINT" => Some(endpoint_url.clone()), + "AWS_ACCESS_KEY_ID" => Some("test".into()), + "AWS_SECRET_ACCESS_KEY" => Some("test".into()), + _ => None, + }); + AwsSecretsManagerV2::load_aws_secret_manager( + Some(true), + KeyManagementSettings { + aws_region_name: Some("us-east-1".into()), + ..Default::default() + }, + environment, + ) + .unwrap() + .unwrap() +} + +#[fixture] +fn default_settings() -> KeyManagementSettings { + KeyManagementSettings::default() +} + +#[rstest] #[case::string_value("KEY", Some("value"))] #[case::missing_value("missing", None)] #[case::non_string_value("BOOL", None)] #[tokio::test] async fn primary_lookup_preserves_read_semantics( + default_settings: KeyManagementSettings, #[case] name: &str, #[case] expected: Option<&str>, ) { @@ -45,7 +82,7 @@ async fn primary_lookup_preserves_read_semantics( .expect(1) .mount(&server) .await; - let manager = manager(&server, KeyManagementSettings::default()); + let manager = manager(&server, default_settings); assert_eq!( manager .read_secret_for_resolver(name, Some("primary"), &|_: &str| None) @@ -57,16 +94,19 @@ async fn primary_lookup_preserves_read_semantics( ); } -#[rstest::rstest] +#[rstest] #[case::access_key("AWS_ACCESS_KEY_ID")] #[case::secret_access_key("AWS_SECRET_ACCESS_KEY")] #[case::region_name("AWS_REGION_NAME")] #[case::region("AWS_REGION")] #[case::bedrock_endpoint("AWS_BEDROCK_RUNTIME_ENDPOINT")] #[tokio::test] -async fn bootstrap_keys_bypass_primary_lookup(#[case] name: &str) { +async fn bootstrap_keys_bypass_primary_lookup( + default_settings: KeyManagementSettings, + #[case] name: &str, +) { let server = MockServer::start().await; - let manager = manager(&server, KeyManagementSettings::default()); + let manager = manager(&server, default_settings); assert_eq!( manager .read_secret_for_resolver(name, Some("primary"), &|_: &str| Some("bootstrap".into())) @@ -79,8 +119,11 @@ async fn bootstrap_keys_bypass_primary_lookup(#[case] name: &str) { ); } +#[rstest] #[tokio::test] -async fn failed_read_returns_none_but_invalid_primary_json_is_an_error() { +async fn failed_read_returns_none_but_invalid_primary_json_is_an_error( + default_settings: KeyManagementSettings, +) { let server = MockServer::start().await; Mock::given(body_partial_json(json!({"SecretId":"missing"}))) .respond_with( @@ -92,7 +135,11 @@ async fn failed_read_returns_none_but_invalid_primary_json_is_an_error() { .respond_with(ResponseTemplate::new(200).set_body_json(json!({"SecretString":"not-json"}))) .mount(&server) .await; - let manager = manager(&server, KeyManagementSettings::default()); + Mock::given(body_partial_json(json!({"SecretId":"no-string"}))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"Name":"no-string"}))) + .mount(&server) + .await; + let manager = manager(&server, default_settings); assert!( manager .async_read_secret("missing") @@ -106,10 +153,17 @@ async fn failed_read_returns_none_but_invalid_primary_json_is_an_error() { .await, Err(Error::PrimarySecret) )); + assert!(matches!( + manager.async_read_secret("no-string").await, + Err(Error::MissingString) + )); } +#[rstest] #[tokio::test] -async fn same_name_rotation_uses_put_and_returns_its_response() { +async fn same_name_rotation_uses_put_and_returns_its_response( + default_settings: KeyManagementSettings, +) { let server = MockServer::start().await; Mock::given(header("x-amz-target", "secretsmanager.PutSecretValue")) .and(body_partial_json( @@ -121,7 +175,7 @@ async fn same_name_rotation_uses_put_and_returns_its_response() { .expect(1) .mount(&server) .await; - let response = manager(&server, KeyManagementSettings::default()) + let response = manager(&server, default_settings) .async_rotate_secret("key", "key", &SecretValue::new("replacement")) .await .unwrap(); @@ -132,8 +186,11 @@ async fn same_name_rotation_uses_put_and_returns_its_response() { assert_eq!(server.received_requests().await.unwrap().len(), 1); } +#[rstest] #[tokio::test] -async fn renamed_rotation_reads_creates_verifies_then_deletes() { +async fn renamed_rotation_reads_creates_verifies_then_deletes( + default_settings: KeyManagementSettings, +) { let server = MockServer::start().await; let step = AtomicUsize::new(0); Mock::given(wiremock::matchers::method("POST")) @@ -176,7 +233,7 @@ async fn renamed_rotation_reads_creates_verifies_then_deletes() { .mount(&server) .await; assert!(matches!( - manager(&server, KeyManagementSettings::default()) + manager(&server, default_settings) .async_rotate_secret("old", "new", &SecretValue::new("replacement")) .await .unwrap(), @@ -184,6 +241,7 @@ async fn renamed_rotation_reads_creates_verifies_then_deletes() { )); } +#[rstest] #[tokio::test] async fn creation_passes_tags_and_kms_and_survives_replication_failure() { let server = MockServer::start().await; @@ -230,6 +288,112 @@ async fn creation_passes_tags_and_kms_and_survives_replication_failure() { ); } +#[rstest] +#[tokio::test] +async fn trait_write_uses_typed_write_context(default_settings: KeyManagementSettings) { + let server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.CreateSecret")) + .and(body_partial_json(json!({ + "Name": "key", + "SecretString": "value", + "Description": "created by caller", + "Tags": [{"Key": "stage", "Value": "test"}], + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"Name": "key"}))) + .expect(1) + .mount(&server) + .await; + let context = SecretWriteContext { + description: Some("created by caller".into()), + tags: std::collections::BTreeMap::from([("stage".into(), "test".into())]), + ..Default::default() + }; + let response = BaseSecretManager::async_write_secret( + &manager(&server, default_settings), + "key", + &SecretValue::new("value"), + &context, + ) + .await + .unwrap(); + assert_eq!(response.name(), Some("key")); +} + +#[rstest] +#[tokio::test] +async fn trait_delete_accepts_an_unspecified_recovery_window( + default_settings: KeyManagementSettings, +) { + let server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.DeleteSecret")) + .and(body_partial_json(json!({"SecretId": "key"}))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"Name": "key"}))) + .expect(1) + .mount(&server) + .await; + let response = BaseSecretManager::async_delete_secret( + &manager(&server, default_settings), + "key", + None, + &SecretOperationContext::default(), + ) + .await + .unwrap(); + assert_eq!(response.name(), Some("key")); +} + +#[rstest] +#[tokio::test] +async fn trait_read_uses_the_aws_region_from_its_operation_context() { + let server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue")) + .respond_with(|request: &wiremock::Request| { + let authorization = request + .headers + .get("authorization") + .unwrap() + .to_str() + .unwrap(); + assert!(authorization.contains("/us-west-2/secretsmanager/aws4_request")); + ResponseTemplate::new(200).set_body_json(json!({"SecretString": "value"})) + }) + .expect(1) + .mount(&server) + .await; + let context = SecretOperationContext::Aws(AwsOperationContext { + region_name: Some("us-west-2".into()), + ..Default::default() + }); + let value = BaseSecretManager::async_read_secret(&loaded_manager(&server), "key", &context) + .await + .unwrap(); + assert_eq!(value.unwrap().expose(), "value"); +} + +#[rstest] +#[tokio::test] +async fn trait_read_applies_the_aws_operation_timeout() { + let server = MockServer::start().await; + Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue")) + .respond_with( + ResponseTemplate::new(200) + .set_delay(Duration::from_secs(1)) + .set_body_json(json!({"SecretString": "late"})), + ) + .expect(1) + .mount(&server) + .await; + let context = SecretOperationContext::Aws(AwsOperationContext { + timeout: Some(Duration::from_millis(30)), + ..Default::default() + }); + assert!(matches!( + BaseSecretManager::async_read_secret(&loaded_manager(&server), "key", &context).await, + Err(Error::Timeout) + )); +} + +#[rstest] #[tokio::test] async fn credential_failures_are_not_swallowed_as_missing_secrets() { use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future}; @@ -260,9 +424,9 @@ async fn credential_failures_are_not_swallowed_as_missing_secrets() { assert!(server.received_requests().await.unwrap().is_empty()); } +#[rstest] #[tokio::test] async fn read_timeout_is_an_error_and_cannot_be_mistaken_for_missing() { - use std::time::Duration; let server = MockServer::start().await; Mock::given(wiremock::matchers::method("POST")) .respond_with( @@ -291,12 +455,16 @@ async fn read_timeout_is_an_error_and_cannot_be_mistaken_for_missing() { )); } -#[rstest::rstest] +#[rstest] #[case::denied(400, "AccessDeniedException")] #[case::throttled(400, "ThrottlingException")] #[case::unavailable(503, "ServiceUnavailableException")] #[tokio::test] -async fn service_failures_remain_errors(#[case] status: u16, #[case] code: &str) { +async fn service_failures_remain_errors( + default_settings: KeyManagementSettings, + #[case] status: u16, + #[case] code: &str, +) { let server = MockServer::start().await; Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue")) .respond_with(ResponseTemplate::new(status).set_body_json(json!({"__type":code}))) @@ -304,7 +472,7 @@ async fn service_failures_remain_errors(#[case] status: u16, #[case] code: &str) .mount(&server) .await; assert!(matches!( - manager(&server, KeyManagementSettings::default()) + manager(&server, default_settings) .async_read_secret("key") .await, Err(Error::Read(_)) diff --git a/litellm-rust/crates/secrets-azure/src/key_vault.rs b/litellm-rust/crates/secrets-azure/src/key_vault.rs index e12289b83f5..81ad0331ca7 100644 --- a/litellm-rust/crates/secrets-azure/src/key_vault.rs +++ b/litellm-rust/crates/secrets-azure/src/key_vault.rs @@ -76,10 +76,7 @@ impl AzureKeyVault { .unwrap_or_default() } - pub async fn get_secret_from_azure_key_vault( - &self, - name: &str, - ) -> Result, Error> { + pub async fn get_secret(&self, name: &str) -> Result, Error> { let token = self .auth .get_azure_ad_token(&self.inputs, &|key| self.environment.get(key)) diff --git a/litellm-rust/crates/secrets-azure/tests/key_vault.rs b/litellm-rust/crates/secrets-azure/tests/key_vault.rs index cf9102d0b45..3b199f81264 100644 --- a/litellm-rust/crates/secrets-azure/tests/key_vault.rs +++ b/litellm-rust/crates/secrets-azure/tests/key_vault.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use litellm_secrets_azure::{AzureKeyVault, Error}; use litellm_secrets_types::{Secret, SecretValue}; +use rstest::{fixture, rstest}; use serde::Deserialize; use wiremock::{ Mock, MockServer, ResponseTemplate, @@ -17,6 +18,7 @@ fn manager(server: &MockServer) -> AzureKeyVault { .unwrap() } +#[rstest] #[tokio::test] async fn reads_secret_with_bearer_token_and_api_version() { let server = MockServer::start().await; @@ -32,7 +34,7 @@ async fn reads_secret_with_bearer_token_and_api_version() { .await; let secret = manager(&server) - .get_secret_from_azure_key_vault("OPENAI-API-KEY") + .get_secret("OPENAI-API-KEY") .await .unwrap() .unwrap(); @@ -40,6 +42,7 @@ async fn reads_secret_with_bearer_token_and_api_version() { assert_eq!(secret, Secret::String(SecretValue::new("s3cret"))); } +#[rstest] #[tokio::test] async fn percent_encodes_secret_name_path_segment() { let server = MockServer::start().await; @@ -53,7 +56,7 @@ async fn percent_encodes_secret_name_path_segment() { .await; let secret = manager(&server) - .get_secret_from_azure_key_vault("name/with spaces") + .get_secret("name/with spaces") .await .unwrap() .unwrap(); @@ -61,9 +64,12 @@ async fn percent_encodes_secret_name_path_segment() { assert_eq!(secret.as_str(), Some("value")); } -#[rstest::rstest] +#[rstest] #[case::not_found(404, None)] +#[case::unauthorized(401, Some(401))] #[case::forbidden(403, Some(403))] +#[case::throttled(429, Some(429))] +#[case::server_error(500, Some(500))] #[tokio::test] async fn handles_statuses(#[case] status: u16, #[case] expected_status: Option) { let server = MockServer::start().await; @@ -73,9 +79,7 @@ async fn handles_statuses(#[case] status: u16, #[case] expected_status: Option assert_eq!(result.unwrap(), None), @@ -83,6 +87,7 @@ async fn handles_statuses(#[case] status: u16, #[case] expected_status: Option, + #[case] missing_environment: bool, +) { + let result = AzureKeyVault::new(Arc::new(move |name: &str| { + (name == "AZURE_KEY_VAULT_URI") + .then(|| uri.map(str::to_owned)) + .flatten() + })); + + if missing_environment { + assert!(matches!( + result, + Err(Error::MissingEnvironment("AZURE_KEY_VAULT_URI")) + )); + } else { + assert!(matches!(result, Err(Error::VaultUri))); + } } -#[rstest::rstest] -#[case("https://myvault.vault.azure.net", "https://vault.azure.net/.default")] -#[case( +#[rstest] +#[case::public_cloud("https://myvault.vault.azure.net", "https://vault.azure.net/.default")] +#[case::government_cloud( "https://v.vault.usgovcloudapi.net/", "https://vault.usgovcloudapi.net/.default" )] -#[case("http://localhost:8080", "https://localhost/.default")] -#[test] +#[case::local("http://localhost:8080", "https://localhost/.default")] fn derives_scope_from_vault_host(#[case] uri: &str, #[case] expected: &str) { let manager = AzureKeyVault::with_client( reqwest::Client::new(), @@ -139,6 +146,7 @@ fn derives_scope_from_vault_host(#[case] uri: &str, #[case] expected: &str) { assert_eq!(manager.scope(), expected); } +#[rstest] #[tokio::test] async fn missing_credentials_do_not_request_vault() { let server = MockServer::start().await; @@ -150,7 +158,7 @@ async fn missing_credentials_do_not_request_vault() { assert!( manager_without_credentials(&server) - .get_secret_from_azure_key_vault("NAME") + .get_secret("NAME") .await .is_err() ); @@ -192,11 +200,15 @@ struct FixtureExpected { error: Option, } +#[fixture] +fn parity_fixture() -> Fixture { + serde_json::from_str(include_str!("fixtures/key_vault_parity.json")).unwrap() +} + +#[rstest] #[tokio::test] -async fn parity_fixture_matches_python_backend_contract() { - let fixture: Fixture = - serde_json::from_str(include_str!("fixtures/key_vault_parity.json")).unwrap(); - for case in fixture.cases { +async fn parity_fixture_matches_python_backend_contract(parity_fixture: Fixture) { + for case in parity_fixture.cases { let server = MockServer::start().await; Mock::given(path(format!("/secrets/{}", case.secret_name))) .respond_with( @@ -205,9 +217,7 @@ async fn parity_fixture_matches_python_backend_contract() { .expect(1) .mount(&server) .await; - let result = manager(&server) - .get_secret_from_azure_key_vault(&case.secret_name) - .await; + let result = manager(&server).get_secret(&case.secret_name).await; if case.expected.missing == Some(true) { assert_eq!(result.unwrap(), None); } else if case.expected.error == Some(true) { diff --git a/litellm-rust/crates/secrets-azure/tests/live.rs b/litellm-rust/crates/secrets-azure/tests/live.rs index a062ba95070..18306382613 100644 --- a/litellm-rust/crates/secrets-azure/tests/live.rs +++ b/litellm-rust/crates/secrets-azure/tests/live.rs @@ -3,18 +3,16 @@ use std::sync::Arc; use litellm_core_utils::settings::ProcessEnvironment; use litellm_secrets_azure::AzureKeyVault; use litellm_secrets_types::Secret; +use rstest::rstest; +#[rstest] #[tokio::test] #[ignore] async fn reads_a_real_secret() { let environment = Arc::new(ProcessEnvironment); let manager = AzureKeyVault::new(environment).unwrap(); let name = std::env::var("AZURE_KEY_VAULT_LIVE_SECRET_NAME").unwrap(); - let secret = manager - .get_secret_from_azure_key_vault(&name) - .await - .unwrap() - .unwrap(); + let secret = manager.get_secret(&name).await.unwrap().unwrap(); assert!(matches!(&secret, Secret::String(_))); let host = std::env::var("AZURE_KEY_VAULT_URI") .unwrap() diff --git a/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs b/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs index 9d6eaaf1c4e..80e6fb12f9e 100644 --- a/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs @@ -2,7 +2,10 @@ use std::{fs, sync::Arc, time::Duration}; use base64::{Engine, engine::general_purpose::STANDARD}; use litellm_core_utils::settings::Lookup; -use litellm_secrets_types::{BaseSecretManager, SecretValue, validate_secret_name}; +use litellm_secrets_types::{ + BaseSecretManager, SecretOperationContext, SecretValue, SecretWriteContext, + validate_secret_name, +}; use moka::future::Cache; use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode}; @@ -140,7 +143,7 @@ impl CyberArkSecretManager { .map_err(|_| Error::Endpoint) } - async fn authenticate(&self) -> Result { + async fn authenticate(&self, context: &SecretOperationContext) -> Result { if let Some(token) = self.token.get(&()).await { return Ok(token); } @@ -155,12 +158,12 @@ impl CyberArkSecretManager { self.account, self.username )) .map_err(|_| Error::Endpoint)?; - let response = self - .client - .post(url) - .body(self.api_key.expose().to_owned()) - .send() - .await?; + let response = with_timeout( + self.client.post(url).body(self.api_key.expose().to_owned()), + context, + ) + .send() + .await?; if !response.status().is_success() { return Err(Error::AuthStatus(response.status().as_u16())); } @@ -169,23 +172,37 @@ impl CyberArkSecretManager { Ok(token) } - async fn authorization_header(&self) -> Result { + async fn authorization_header( + &self, + context: &SecretOperationContext, + ) -> Result { Ok(format!( "Token token=\"{}\"", - self.authenticate().await?.expose() + self.authenticate(context).await?.expose() )) } pub async fn async_read_secret(&self, name: &str) -> Result, Error> { + self.async_read_secret_with_context(name, &SecretOperationContext::default()) + .await + } + + pub async fn async_read_secret_with_context( + &self, + name: &str, + context: &SecretOperationContext, + ) -> Result, Error> { if let Some(value) = self.secrets.get(name).await { return Ok(Some(value)); } - let response = self - .client - .get(self.secret_url(name)?) - .header("Authorization", self.authorization_header().await?) - .send() - .await?; + let response = with_timeout( + self.client + .get(self.secret_url(name)?) + .header("Authorization", self.authorization_header(context).await?), + context, + ) + .send() + .await?; if response.status() == reqwest::StatusCode::NOT_FOUND { return Ok(None); } @@ -198,20 +215,38 @@ impl CyberArkSecretManager { } pub async fn async_write_secret( + &self, + name: &str, + value: &SecretValue, + description: Option<&str>, + ) -> Result<(), Error> { + self.async_write_secret_with_context( + name, + value, + description, + &SecretOperationContext::default(), + ) + .await + } + + pub async fn async_write_secret_with_context( &self, name: &str, value: &SecretValue, _description: Option<&str>, + context: &SecretOperationContext, ) -> Result<(), Error> { validate_secret_name(name)?; - self.ensure_variable_exists(name).await; - let response = self - .client - .post(self.secret_url(name)?) - .header("Authorization", self.authorization_header().await?) - .body(value.expose().to_owned()) - .send() - .await?; + self.ensure_variable_exists(name, context).await; + let response = with_timeout( + self.client + .post(self.secret_url(name)?) + .header("Authorization", self.authorization_header(context).await?) + .body(value.expose().to_owned()), + context, + ) + .send() + .await?; if !response.status().is_success() { return Err(Error::Status(response.status().as_u16())); } @@ -219,7 +254,7 @@ impl CyberArkSecretManager { Ok(()) } - async fn ensure_variable_exists(&self, name: &str) { + async fn ensure_variable_exists(&self, name: &str, context: &SecretOperationContext) { let policy_url = self .endpoint .join(&format!("policies/{}/policy/root", self.account)); @@ -227,7 +262,7 @@ impl CyberArkSecretManager { tracing::warn!("Could not build CyberArk policy endpoint"); return; }; - let Ok(authorization) = self.authorization_header().await else { + let Ok(authorization) = self.authorization_header(context).await else { tracing::warn!("Could not authenticate while ensuring CyberArk variable exists"); return; }; @@ -235,14 +270,16 @@ impl CyberArkSecretManager { "- !variable {}\n", serde_json::to_string(name).expect("serializing a string cannot fail") ); - let response = self - .client - .post(policy_url) - .header("Authorization", authorization) - .header("Content-Type", "application/x-yaml") - .body(body) - .send() - .await; + let response = with_timeout( + self.client + .post(policy_url) + .header("Authorization", authorization) + .header("Content-Type", "application/x-yaml") + .body(body), + context, + ) + .send() + .await; match response { Ok(response) if response.status().is_success() => {} Ok(response) @@ -271,7 +308,21 @@ impl CyberArkSecretManager { pub async fn async_delete_secret( &self, name: &str, - _recovery_window_in_days: i64, + recovery_window_in_days: Option, + ) -> Result { + self.async_delete_secret_with_context( + name, + recovery_window_in_days, + &SecretOperationContext::default(), + ) + .await + } + + pub async fn async_delete_secret_with_context( + &self, + name: &str, + _recovery_window_in_days: Option, + _context: &SecretOperationContext, ) -> Result { tracing::warn!( "CyberArk Conjur does not support direct secret deletion. Secrets must be removed through policy updates." @@ -286,29 +337,50 @@ impl BaseSecretManager for CyberArkSecretManager { type WriteResponse = (); type DeleteResponse = DeleteOutcome; - async fn async_read_secret(&self, name: &str) -> Result, Error> { - self.async_read_secret(name).await + async fn async_read_secret( + &self, + name: &str, + context: &SecretOperationContext, + ) -> Result, Error> { + self.async_read_secret_with_context(name, context).await } async fn async_write_secret( &self, name: &str, value: &SecretValue, - description: Option<&str>, + context: &SecretWriteContext, ) -> Result<(), Error> { - self.async_write_secret(name, value, description).await + self.async_write_secret_with_context( + name, + value, + context.description.as_deref(), + &context.operation, + ) + .await } async fn async_delete_secret( &self, name: &str, - recovery_window_in_days: i64, + recovery_window_in_days: Option, + context: &SecretOperationContext, ) -> Result { - self.async_delete_secret(name, recovery_window_in_days) + self.async_delete_secret_with_context(name, recovery_window_in_days, context) .await } } +fn with_timeout( + request: reqwest::RequestBuilder, + context: &SecretOperationContext, +) -> reqwest::RequestBuilder { + match context.timeout() { + Some(timeout) => request.timeout(timeout), + None => request, + } +} + fn normalize_endpoint(mut endpoint: reqwest::Url) -> reqwest::Url { if !endpoint.path().ends_with('/') { endpoint.set_path(&format!("{}/", endpoint.path())); diff --git a/litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs b/litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs index fd7198b70fb..59964d90d50 100644 --- a/litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs +++ b/litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs @@ -2,7 +2,10 @@ use std::{sync::Arc, time::Duration}; use base64::{Engine, engine::general_purpose::STANDARD}; use litellm_secrets_cyberark::{CyberArkSecretManager, DeleteOutcome, Error}; -use litellm_secrets_types::SecretValue; +use litellm_secrets_types::{ + BaseSecretManager, CyberarkOperationContext, SecretOperationContext, SecretValue, +}; +use rstest::{fixture, rstest}; use serde::Deserialize; use wiremock::{ Match, Mock, MockServer, Request, ResponseTemplate, @@ -40,7 +43,8 @@ impl Match for RawPath { } } -fn fixture() -> ParityFixture { +#[fixture] +fn parity_fixture() -> ParityFixture { serde_json::from_str(include_str!("fixtures/parity.json")).unwrap() } @@ -65,6 +69,7 @@ async fn mount_auth(server: &MockServer, expected: u64) { .await; } +#[rstest] #[tokio::test] async fn successful_reads_cache_auth_secret_and_redact_values() { let server = MockServer::start().await; @@ -89,6 +94,7 @@ async fn successful_reads_cache_auth_secret_and_redact_values() { } } +#[rstest] #[tokio::test] async fn concurrent_reads_share_authentication_request() { let server = MockServer::start().await; @@ -122,7 +128,7 @@ async fn concurrent_reads_share_authentication_request() { assert_eq!(second.unwrap().unwrap().expose(), "value"); } -#[rstest::rstest] +#[rstest] #[case::not_found(404)] #[case::unauthorized(401)] #[case::forbidden(403)] @@ -162,6 +168,7 @@ async fn failed_reads_are_not_cached(#[case] status: u16) { } } +#[rstest] #[tokio::test] async fn failed_authentication_is_not_cached_and_does_not_read_secret() { let server = MockServer::start().await; @@ -199,6 +206,29 @@ async fn failed_authentication_is_not_cached_and_does_not_read_secret() { ); } +#[rstest] +#[tokio::test] +async fn trait_read_applies_cyberark_operation_timeout_to_authentication() { + let server = MockServer::start().await; + Mock::given(path("/authn/acct/admin/authenticate")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(TOKEN_JSON) + .set_delay(Duration::from_millis(50)), + ) + .mount(&server) + .await; + let manager = manager(&server, Duration::from_secs(60)); + let context = SecretOperationContext::Cyberark(CyberarkOperationContext { + timeout: Some(Duration::from_millis(10)), + }); + + let result = BaseSecretManager::async_read_secret(&manager, "key", &context).await; + + assert!(matches!(result, Err(Error::Http(error)) if error.is_timeout())); +} + +#[rstest] #[tokio::test] async fn expired_tokens_and_secrets_are_fetched_again() { let server = MockServer::start().await; @@ -215,13 +245,14 @@ async fn expired_tokens_and_secrets_are_fetched_again() { } } -#[rstest::rstest] +#[rstest] +#[case::plain("OPENAI_API_KEY")] +#[case::path("team/app/key")] +#[case::punctuation("a b+c.d-e_f~g")] +#[case::quote("needs \"quote\"")] #[tokio::test] -async fn secret_names_use_python_quote_encoding( - #[values("OPENAI_API_KEY", "team/app/key", "a b+c.d-e_f~g", "needs \"quote\"")] name: &str, -) { - let fixture = fixture(); - let secret = fixture +async fn secret_names_use_python_quote_encoding(parity_fixture: ParityFixture, #[case] name: &str) { + let secret = parity_fixture .secrets .iter() .find(|secret| secret.name == name) @@ -244,11 +275,11 @@ async fn secret_names_use_python_quote_encoding( ); } -#[rstest::rstest] -#[case(201)] -#[case(409)] -#[case(422)] -#[case(500)] +#[rstest] +#[case::created(201)] +#[case::already_exists(409)] +#[case::unprocessable(422)] +#[case::server_error(500)] #[tokio::test] async fn writes_tolerate_policy_status_and_cache_value(#[case] policy_status: u16) { let server = MockServer::start().await; @@ -282,6 +313,7 @@ async fn writes_tolerate_policy_status_and_cache_value(#[case] policy_status: u1 ); } +#[rstest] #[tokio::test] async fn failed_value_write_is_not_cached() { let server = MockServer::start().await; @@ -319,13 +351,17 @@ async fn failed_value_write_is_not_cached() { ); } +#[rstest] +#[case::parent("../etc")] +#[case::embedded_parent("team/../etc")] +#[case::control("key\n")] #[tokio::test] -async fn unsafe_names_fail_before_http_calls() { +async fn unsafe_names_fail_before_http_calls(#[case] name: &str) { let server = MockServer::start().await; let manager = manager(&server, Duration::from_secs(60)); assert!(matches!( manager - .async_write_secret("../etc", &SecretValue::new("v"), None) + .async_write_secret(name, &SecretValue::new("v"), None) .await, Err(Error::Operation( litellm_secrets_types::Error::UnsafeSecretName @@ -333,6 +369,7 @@ async fn unsafe_names_fail_before_http_calls() { )); } +#[rstest] #[tokio::test] async fn delete_invalidates_cache_and_reports_not_supported() { let server = MockServer::start().await; @@ -353,7 +390,7 @@ async fn delete_invalidates_cache_and_reports_not_supported() { "v" ); assert_eq!( - manager.async_delete_secret("key", 7).await.unwrap(), + manager.async_delete_secret("key", Some(7)).await.unwrap(), DeleteOutcome::NotSupported ); assert_eq!( @@ -367,7 +404,7 @@ async fn delete_invalidates_cache_and_reports_not_supported() { ); } -#[test] +#[rstest] fn new_validates_credentials_before_license_and_configuration() { let empty: Arc = Arc::new(|_: &str| None); @@ -413,6 +450,7 @@ fn new_validates_credentials_before_license_and_configuration() { )); } +#[rstest] #[tokio::test] async fn new_reads_environment_defaults_end_to_end() { let server = MockServer::start().await; @@ -446,7 +484,7 @@ async fn new_reads_environment_defaults_end_to_end() { ); } -#[test] +#[rstest] fn new_reports_missing_client_certificate_files() { assert!(matches!( CyberArkSecretManager::new( @@ -461,6 +499,7 @@ fn new_reports_missing_client_certificate_files() { )); } +#[rstest] #[tokio::test] async fn trailing_slash_endpoint_preserves_base_path() { let server = MockServer::start().await; @@ -494,23 +533,25 @@ async fn trailing_slash_endpoint_preserves_base_path() { ); } -#[test] -fn parity_fixture_matches_authentication_contract() { - let fixture = fixture(); - assert_eq!(fixture.endpoint, "http://conjur.test:8080"); - assert_eq!(fixture.account, "acct"); - assert_eq!(fixture.username, "admin"); - assert_eq!(fixture.api_key, "k3y"); - assert_eq!(fixture.authenticate_path, "/authn/acct/admin/authenticate"); - assert_eq!(fixture.token_json, TOKEN_JSON); +#[rstest] +fn parity_fixture_matches_authentication_contract(parity_fixture: ParityFixture) { + assert_eq!(parity_fixture.endpoint, "http://conjur.test:8080"); + assert_eq!(parity_fixture.account, "acct"); + assert_eq!(parity_fixture.username, "admin"); + assert_eq!(parity_fixture.api_key, "k3y"); assert_eq!( - fixture.authorization_header, + parity_fixture.authenticate_path, + "/authn/acct/admin/authenticate" + ); + assert_eq!(parity_fixture.token_json, TOKEN_JSON); + assert_eq!( + parity_fixture.authorization_header, format!("Token token=\"{}\"", STANDARD.encode(TOKEN_JSON)) ); - assert_eq!(fixture.policy_path, "/policies/acct/policy/root"); - assert_eq!(fixture.secrets.len(), 4); + assert_eq!(parity_fixture.policy_path, "/policies/acct/policy/root"); + assert_eq!(parity_fixture.secrets.len(), 4); assert_eq!( - fixture.secrets[1].policy_body, + parity_fixture.secrets[1].policy_body, "- !variable \"team/app/key\"\n" ); } diff --git a/litellm-rust/crates/secrets-google/tests/kms.rs b/litellm-rust/crates/secrets-google/tests/kms.rs index 667ecd268c8..9739a46b10c 100644 --- a/litellm-rust/crates/secrets-google/tests/kms.rs +++ b/litellm-rust/crates/secrets-google/tests/kms.rs @@ -1,11 +1,14 @@ use base64::{Engine, engine::general_purpose::STANDARD}; use google_cloud_kms_v1::client::KeyManagementService; -use litellm_secrets_google::GoogleKms; +use litellm_core_utils::settings::Lookup; +use litellm_secrets_google::{Error, GoogleKms, kms::validate_environment}; +use rstest::rstest; use wiremock::{ Mock, MockServer, ResponseTemplate, matchers::{body_json, path}, }; +#[rstest] #[tokio::test] async fn google_kms_decrypts_using_the_configured_resource() { let server = MockServer::start().await; @@ -35,15 +38,49 @@ async fn google_kms_decrypts_using_the_configured_resource() { ); } +#[rstest] +#[case::unset(None)] +#[case::disabled(Some(false))] #[tokio::test] -async fn disabled_google_kms_loader_does_not_require_environment_configuration() { +async fn disabled_google_kms_loader_does_not_require_environment_configuration( + #[case] enabled: Option, +) { use std::sync::Arc; - for enabled in [None, Some(false)] { - assert!( - litellm_secrets_google::load_google_kms(enabled, Arc::new(|_: &str| None)) - .await - .unwrap() - .is_none() - ); - } + assert!( + litellm_secrets_google::load_google_kms(enabled, Arc::new(|_: &str| None)) + .await + .unwrap() + .is_none() + ); +} + +#[rstest] +#[case::credentials_missing(None, None, "GOOGLE_APPLICATION_CREDENTIALS")] +#[case::resource_missing(Some("credentials"), None, "GOOGLE_KMS_RESOURCE_NAME")] +fn enabled_google_kms_requires_all_environment_values( + #[case] credentials: Option<&str>, + #[case] resource: Option<&str>, + #[case] missing: &'static str, +) { + let environment = move |name: &str| match name { + "GOOGLE_APPLICATION_CREDENTIALS" => credentials.map(str::to_owned), + "GOOGLE_KMS_RESOURCE_NAME" => resource.map(str::to_owned), + _ => None, + }; + + assert!(matches!( + validate_environment(&environment as &dyn Lookup), + Err(Error::MissingEnvironment(name)) if name == missing + )); +} + +#[rstest] +fn complete_google_kms_environment_is_valid() { + let environment = |name: &str| match name { + "GOOGLE_APPLICATION_CREDENTIALS" => Some("credentials".to_owned()), + "GOOGLE_KMS_RESOURCE_NAME" => Some("resource".to_owned()), + _ => None, + }; + + assert!(validate_environment(&environment as &dyn Lookup).is_ok()); } diff --git a/litellm-rust/crates/secrets-google/tests/secret_manager.rs b/litellm-rust/crates/secrets-google/tests/secret_manager.rs index b3b1d29e62c..867dcf935c1 100644 --- a/litellm-rust/crates/secrets-google/tests/secret_manager.rs +++ b/litellm-rust/crates/secrets-google/tests/secret_manager.rs @@ -2,6 +2,7 @@ use std::{sync::Arc, time::Duration}; use base64::{Engine, engine::general_purpose::STANDARD}; use litellm_secrets_google::{Error, GoogleSecretManager}; +use rstest::{fixture, rstest}; use wiremock::{ Mock, MockServer, ResponseTemplate, @@ -20,11 +21,17 @@ fn manager(server: &MockServer, always_read: bool, ttl: Duration) -> GoogleSecre .unwrap() } -#[rstest::rstest] +#[fixture] +fn default_ttl() -> Duration { + Duration::from_secs(60) +} + +#[rstest] #[case::nonempty("private-value")] #[case::empty("")] #[tokio::test] async fn successful_reads_use_auth_latest_version_and_cache_including_empty_values( + default_ttl: Duration, #[case] value: &str, ) { let server = MockServer::start().await; @@ -39,7 +46,7 @@ async fn successful_reads_use_auth_latest_version_and_cache_including_empty_valu .expect(1) .mount(&server) .await; - let manager = manager(&server, false, Duration::from_secs(60)); + let manager = manager(&server, false, default_ttl); for _ in 0..2 { assert_eq!( manager @@ -54,21 +61,44 @@ async fn successful_reads_use_auth_latest_version_and_cache_including_empty_valu } } -#[rstest::rstest] -#[case::not_found(404, serde_json::json!({}))] -#[case::unauthorized(401, serde_json::json!({}))] -#[case::forbidden(403, serde_json::json!({}))] -#[case::throttled(429, serde_json::json!({}))] -#[case::unavailable(503, serde_json::json!({}))] -#[case::missing_payload(200, serde_json::json!({"payload":{}}))] -#[case::invalid_base64(200, serde_json::json!({"payload":{"data":"%%%"}}))] +enum ExpectedReadFailure { + Missing, + Status(u16), + MissingPayload, + Base64, + Utf8, +} + +#[rstest] +#[case::not_found(404, serde_json::json!({}), ExpectedReadFailure::Missing)] +#[case::unauthorized(401, serde_json::json!({}), ExpectedReadFailure::Status(401))] +#[case::forbidden(403, serde_json::json!({}), ExpectedReadFailure::Status(403))] +#[case::throttled(429, serde_json::json!({}), ExpectedReadFailure::Status(429))] +#[case::unavailable(503, serde_json::json!({}), ExpectedReadFailure::Status(503))] +#[case::missing_payload( + 200, + serde_json::json!({"payload":{}}), + ExpectedReadFailure::MissingPayload +)] +#[case::invalid_base64( + 200, + serde_json::json!({"payload":{"data":"%%%"}}), + ExpectedReadFailure::Base64 +)] +#[case::invalid_utf8( + 200, + serde_json::json!({"payload":{"data":STANDARD.encode([0xff])}}), + ExpectedReadFailure::Utf8 +)] #[tokio::test] async fn failed_or_missing_reads_are_not_cached( + default_ttl: Duration, #[case] status: u16, #[case] body: serde_json::Value, + #[case] expected: ExpectedReadFailure, ) { let server = MockServer::start().await; - let manager = manager(&server, false, Duration::from_secs(60)); + let manager = manager(&server, false, default_ttl); let failing = Mock::given(path( "/v1/projects/project/secrets/key/versions/latest:access", )) @@ -77,13 +107,16 @@ async fn failed_or_missing_reads_are_not_cached( .mount_as_scoped(&server) .await; let result = manager.get_secret_from_google_secret_manager("key").await; - match status { - 404 => assert_eq!(result.unwrap(), None), - 200 => assert!(matches!( - result, - Err(Error::MissingPayload | Error::Base64(_)) - )), - status => assert!(matches!(result, Err(Error::Status(actual)) if actual == status)), + match expected { + ExpectedReadFailure::Missing => assert_eq!(result.unwrap(), None), + ExpectedReadFailure::Status(expected) => { + assert!(matches!(result, Err(Error::Status(actual)) if actual == expected)); + } + ExpectedReadFailure::MissingPayload => { + assert!(matches!(result, Err(Error::MissingPayload))); + } + ExpectedReadFailure::Base64 => assert!(matches!(result, Err(Error::Base64(_)))), + ExpectedReadFailure::Utf8 => assert!(matches!(result, Err(Error::Utf8))), } drop(failing); Mock::given(path( @@ -109,7 +142,7 @@ async fn failed_or_missing_reads_are_not_cached( } } -#[rstest::rstest] +#[rstest] #[case::always_read(true, Duration::from_secs(60))] #[case::expired_cache(false, Duration::from_millis(1))] #[tokio::test] @@ -141,7 +174,7 @@ async fn always_read_and_expired_cache_fetch_again( } } -#[test] +#[rstest] fn google_manager_requires_host_license_and_project_configuration() { assert!(matches!( GoogleSecretManager::new(Arc::new(|_: &str| None), false), @@ -155,13 +188,29 @@ fn google_manager_requires_host_license_and_project_configuration() { )); } -#[rstest::rstest] -#[case("true")] -#[case("null")] -#[case("\"text\"")] -#[case("{\"key\":1}")] +#[rstest] +#[case::provider_specific("GOOGLE_SECRET_MANAGER_REFRESH_INTERVAL")] +#[case::shared("SECRET_MANAGER_REFRESH_INTERVAL")] +fn google_manager_rejects_invalid_refresh_intervals(#[case] variable: &'static str) { + let environment = Arc::new(move |name: &str| match name { + "GOOGLE_SECRET_MANAGER_PROJECT_ID" => Some("project".to_owned()), + name if name == variable => Some("not-a-number".to_owned()), + _ => None, + }); + + assert!(matches!( + GoogleSecretManager::new(environment, true), + Err(Error::RefreshInterval) + )); +} + +#[rstest] +#[case::boolean("true")] +#[case::null("null")] +#[case::string("\"text\"")] +#[case::object("{\"key\":1}")] #[tokio::test] -async fn cache_preserves_raw_values(#[case] raw: &str) { +async fn cache_preserves_raw_values(default_ttl: Duration, #[case] raw: &str) { let server = MockServer::start().await; Mock::given(path( "/v1/projects/project/secrets/key/versions/latest:access", @@ -173,7 +222,7 @@ async fn cache_preserves_raw_values(#[case] raw: &str) { .expect(1) .mount(&server) .await; - let manager = manager(&server, false, Duration::from_secs(60)); + let manager = manager(&server, false, default_ttl); for _ in 0..2 { assert_eq!( manager diff --git a/litellm-rust/crates/secrets-hashicorp/src/error.rs b/litellm-rust/crates/secrets-hashicorp/src/error.rs index e26033af085..664f9ab18dd 100644 --- a/litellm-rust/crates/secrets-hashicorp/src/error.rs +++ b/litellm-rust/crates/secrets-hashicorp/src/error.rs @@ -4,6 +4,8 @@ pub enum Error { EnterpriseRequired, #[error("invalid secret name")] InvalidSecretName(#[from] litellm_secrets_types::Error), + #[error("HashiCorp Vault received an incompatible operation context")] + InvalidOperationContext, #[error("HashiCorp Vault client failed")] Client( #[from] @@ -29,6 +31,8 @@ pub enum Error { MalformedPayload, #[error("HashiCorp Vault secret value is not a string")] NonStringValue, + #[error("HashiCorp Vault operation timed out")] + Timeout, #[error("invalid HashiCorp Vault refresh interval")] RefreshInterval, } diff --git a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs index 3ad9a549438..a4d1efb4ff6 100644 --- a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs @@ -1,13 +1,15 @@ use std::{ collections::HashMap, fmt, + future::Future, sync::Arc, time::{Duration, Instant}, }; use litellm_core_utils::settings::Lookup; use litellm_secrets_types::{ - BaseSecretManager, SecretValue, async_rotate_secret, validate_secret_name, + BaseSecretManager, HashicorpOperationContext, SecretOperationContext, SecretValue, + SecretWriteContext, async_rotate_secret, validate_secret_name, }; use moka::future::Cache; use rustify::errors::ClientError as RustifyClientError; @@ -31,17 +33,23 @@ struct CachedClient { expires_at: Option, } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, Hash, PartialEq, Eq)] pub struct SecretLocation { pub namespace: Option, pub mount: String, pub path: String, } +#[derive(Clone, Hash, PartialEq, Eq)] +struct CacheKey { + location: SecretLocation, + data_key: String, +} + #[derive(Clone)] pub struct HashicorpVault { config: HashicorpVaultConfig, - cache: Cache, + cache: Cache, auth_client: Arc>>, } @@ -71,7 +79,7 @@ impl HashicorpVault { if !enterprise_enabled { return Err(Error::EnterpriseRequired); } - let cache: Cache = Cache::builder() + let cache: Cache = Cache::builder() .max_capacity(CACHE_CAPACITY) .time_to_live(config.refresh_interval) .build(); @@ -83,9 +91,21 @@ impl HashicorpVault { } pub fn secret_location(&self, secret_name: &str) -> Result { + self.secret_location_with_context(secret_name, &SecretOperationContext::default()) + } + + pub fn secret_location_with_context( + &self, + secret_name: &str, + context: &SecretOperationContext, + ) -> Result { validate_secret_name(secret_name).map_err(Error::InvalidSecretName)?; + let operation: Option<&HashicorpOperationContext> = hashicorp_context(context)?; let path: String = [ - self.config.path_prefix.clone(), + operation + .and_then(|operation| operation.path_prefix.as_deref()) + .and_then(path_component) + .or_else(|| self.config.path_prefix.clone()), Some(secret_name.to_owned()), ] .into_iter() @@ -94,7 +114,10 @@ impl HashicorpVault { .join("/"); Ok(SecretLocation { namespace: self.config.secret_namespace().map(str::to_owned), - mount: self.config.mount.clone(), + mount: operation + .and_then(|operation| operation.mount.as_deref()) + .and_then(path_component) + .unwrap_or_else(|| self.config.mount.clone()), path, }) } @@ -104,19 +127,37 @@ impl HashicorpVault { } pub async fn async_read_secret(&self, secret_name: &str) -> Result, Error> { - let location: SecretLocation = self.secret_location(secret_name)?; - let cache_key: String = cache_key(&location); + self.async_read_secret_with_context(secret_name, &SecretOperationContext::default()) + .await + } + + pub async fn async_read_secret_with_context( + &self, + secret_name: &str, + context: &SecretOperationContext, + ) -> Result, Error> { + let location: SecretLocation = self.secret_location_with_context(secret_name, context)?; + let data_key: String = data_key(context)?; + let cache_key = CacheKey { + location: location.clone(), + data_key: data_key.clone(), + }; if let Some(value) = self.cache.get(&cache_key).await { return Ok(Some(value)); } - let client: Arc = self.vault_client().await?; - let data: HashMap = + let data: Option> = with_timeout(context, async { + let client: Arc = self.vault_client().await?; match kv2::read(client.as_ref(), &location.mount, &location.path).await { - Ok(data) => data, - Err(error) if api_status(&error) == Some(404) => return Ok(None), - Err(error) => return Err(map_api_error(error, ErrorContext::Read)), - }; - let Some(value) = data.get("key") else { + Ok(data) => Ok(Some(data)), + Err(error) if api_status(&error) == Some(404) => Ok(None), + Err(error) => Err(map_api_error(error, ErrorContext::Read)), + } + }) + .await?; + let Some(data) = data else { + return Ok(None); + }; + let Some(value) = data.get(&data_key) else { return Ok(None); }; let value: &str = value.as_str().ok_or(Error::NonStringValue)?; @@ -131,11 +172,29 @@ impl HashicorpVault { value: SecretValue, description: Option<&str>, ) -> Result { - let location: SecretLocation = self.secret_location(secret_name)?; - let cache_key: String = cache_key(&location); - let data: HashMap = match description { + self.async_write_secret_with_context( + secret_name, + &value, + &SecretWriteContext { + description: description.map(str::to_owned), + ..SecretWriteContext::default() + }, + ) + .await + } + + pub async fn async_write_secret_with_context( + &self, + secret_name: &str, + value: &SecretValue, + context: &SecretWriteContext, + ) -> Result { + let location: SecretLocation = + self.secret_location_with_context(secret_name, &context.operation)?; + let data_key: String = data_key(&context.operation)?; + let data: HashMap = match context.description.as_deref() { Some(description) => [ - ("key".to_owned(), Value::String(value.expose().to_owned())), + (data_key, Value::String(value.expose().to_owned())), ( "description".to_owned(), Value::String(description.to_owned()), @@ -143,27 +202,41 @@ impl HashicorpVault { ] .into_iter() .collect(), - None => [("key".to_owned(), Value::String(value.expose().to_owned()))] + None => [(data_key, Value::String(value.expose().to_owned()))] .into_iter() .collect(), }; - let client: Arc = self.vault_client().await?; - let metadata = kv2::set(client.as_ref(), &location.mount, &location.path, &data) - .await - .map_err(|error| map_api_error(error, ErrorContext::Secret))?; - self.cache.invalidate(&cache_key).await; + let metadata = with_timeout(&context.operation, async { + let client: Arc = self.vault_client().await?; + kv2::set(client.as_ref(), &location.mount, &location.path, &data) + .await + .map_err(|error| map_api_error(error, ErrorContext::Secret)) + }) + .await?; + self.cache.invalidate_all(); serde_json::to_value(metadata) .map_err(|source| Error::Client(ClientError::JsonParseError { source })) } pub async fn async_delete_secret(&self, secret_name: &str) -> Result<(), Error> { - let location: SecretLocation = self.secret_location(secret_name)?; - let cache_key: String = cache_key(&location); - let client: Arc = self.vault_client().await?; - kv2::delete_latest(client.as_ref(), &location.mount, &location.path) + self.async_delete_secret_with_context(secret_name, &SecretOperationContext::default()) .await - .map_err(|error| map_api_error(error, ErrorContext::Secret))?; - self.cache.invalidate(&cache_key).await; + } + + pub async fn async_delete_secret_with_context( + &self, + secret_name: &str, + context: &SecretOperationContext, + ) -> Result<(), Error> { + let location: SecretLocation = self.secret_location_with_context(secret_name, context)?; + with_timeout(context, async { + let client: Arc = self.vault_client().await?; + kv2::delete_latest(client.as_ref(), &location.mount, &location.path) + .await + .map_err(|error| map_api_error(error, ErrorContext::Secret)) + }) + .await?; + self.cache.invalidate_all(); Ok(()) } @@ -173,7 +246,23 @@ impl HashicorpVault { new_name: &str, value: &SecretValue, ) -> Result { - async_rotate_secret(self, current_name, new_name, value).await + self.async_rotate_secret_with_context( + current_name, + new_name, + value, + &SecretOperationContext::default(), + ) + .await + } + + pub async fn async_rotate_secret_with_context( + &self, + current_name: &str, + new_name: &str, + value: &SecretValue, + context: &SecretOperationContext, + ) -> Result { + async_rotate_secret(self, current_name, new_name, value, context).await } async fn vault_client(&self) -> Result, Error> { @@ -254,25 +343,30 @@ impl BaseSecretManager for HashicorpVault { type WriteResponse = Value; type DeleteResponse = (); - async fn async_read_secret(&self, name: &str) -> Result, Error> { - HashicorpVault::async_read_secret(self, name).await + async fn async_read_secret( + &self, + name: &str, + context: &SecretOperationContext, + ) -> Result, Error> { + HashicorpVault::async_read_secret_with_context(self, name, context).await } async fn async_write_secret( &self, name: &str, value: &SecretValue, - description: Option<&str>, + context: &SecretWriteContext, ) -> Result { - HashicorpVault::async_write_secret(self, name, value.clone(), description).await + HashicorpVault::async_write_secret_with_context(self, name, value, context).await } async fn async_delete_secret( &self, name: &str, - _recovery_window_in_days: i64, + _recovery_window_in_days: Option, + context: &SecretOperationContext, ) -> Result<(), Error> { - HashicorpVault::async_delete_secret(self, name).await + HashicorpVault::async_delete_secret_with_context(self, name, context).await } } @@ -283,11 +377,42 @@ enum ErrorContext { Secret, } -fn cache_key(location: &SecretLocation) -> String { - format!( - "{:?}/{}/{}", - location.namespace, location.mount, location.path - ) +fn hashicorp_context( + context: &SecretOperationContext, +) -> Result, Error> { + match context { + SecretOperationContext::Hashicorp(context) => Ok(Some(context)), + SecretOperationContext::Default => Ok(None), + SecretOperationContext::Aws(_) | SecretOperationContext::Cyberark(_) => { + Err(Error::InvalidOperationContext) + } + } +} + +fn path_component(value: &str) -> Option { + let value: &str = value.trim().trim_matches('/'); + (!value.is_empty()).then(|| value.to_owned()) +} + +fn data_key(context: &SecretOperationContext) -> Result { + Ok(hashicorp_context(context)? + .and_then(|context| context.data_key.as_deref()) + .map(str::trim) + .filter(|data_key| !data_key.is_empty()) + .map(str::to_owned) + .unwrap_or_else(|| "key".to_owned())) +} + +async fn with_timeout( + context: &SecretOperationContext, + operation: impl Future>, +) -> Result { + match context.timeout() { + Some(timeout) => tokio::time::timeout(timeout, operation) + .await + .map_err(|_| Error::Timeout)?, + None => operation.await, + } } fn identity_for(tls: Option<&TlsCertAuth>) -> Result, Error> { diff --git a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs index c52db46e41e..6b322de4d8d 100644 --- a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs +++ b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs @@ -2,7 +2,11 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; use litellm_core_utils::settings::Lookup; use litellm_secrets_hashicorp::{Error, HashicorpVault, HashicorpVaultConfig}; -use litellm_secrets_types::SecretValue; +use litellm_secrets_types::{ + AwsOperationContext, BaseSecretManager, CyberarkOperationContext, HashicorpOperationContext, + SecretOperationContext, SecretValue, SecretWriteContext, +}; +use rstest::{fixture, rstest}; use serde::Deserialize; use serde_json::json; use wiremock::{ @@ -69,8 +73,14 @@ fn read_response(data: serde_json::Value) -> serde_json::Value { }) } +#[fixture] +fn token_values() -> Vec<(&'static str, &'static str)> { + vec![("HCP_VAULT_TOKEN", "token")] +} + +#[rstest] #[tokio::test] -async fn token_reads_use_vault_headers_and_cache_values() { +async fn token_reads_use_vault_headers_and_cache_values(token_values: Vec<(&str, &str)>) { let server: MockServer = MockServer::start().await; Mock::given(method("GET")) .and(path("/v1/secret/data/name")) @@ -81,7 +91,7 @@ async fn token_reads_use_vault_headers_and_cache_values() { .expect(1) .mount(&server) .await; - let manager: HashicorpVault = manager(&server, &[("HCP_VAULT_TOKEN", "token")]); + let manager: HashicorpVault = manager(&server, &token_values); assert_eq!( manager @@ -109,6 +119,7 @@ async fn token_reads_use_vault_headers_and_cache_values() { ); } +#[rstest] #[tokio::test] async fn namespace_mount_and_prefix_are_sanitized_in_the_url() { let server: MockServer = MockServer::start().await; @@ -138,7 +149,7 @@ async fn namespace_mount_and_prefix_are_sanitized_in_the_url() { assert!(manager.async_read_secret("name").await.unwrap().is_some()); } -#[test] +#[rstest] fn trailing_address_slashes_are_removed() { let environment: Arc = Arc::new(|name: &str| match name { "HCP_VAULT_ADDR" => Some("http://vault.test:8200///".to_owned()), @@ -159,9 +170,9 @@ fn trailing_address_slashes_are_removed() { ); } -#[rstest::rstest] -#[case("-1")] -#[case("not-a-number")] +#[rstest] +#[case::negative("-1")] +#[case::not_a_number("not-a-number")] fn invalid_refresh_intervals_are_rejected(#[case] value: &str) { let environment: Arc = Arc::new(move |name: &str| match name { "HCP_VAULT_REFRESH_INTERVAL" => Some(value.to_owned()), @@ -174,6 +185,7 @@ fn invalid_refresh_intervals_are_rejected(#[case] value: &str) { )); } +#[rstest] #[tokio::test] async fn approle_login_uses_namespace_and_reuses_the_token() { let server: MockServer = MockServer::start().await; @@ -216,6 +228,7 @@ async fn approle_login_uses_namespace_and_reuses_the_token() { assert!(manager.async_read_secret("name-2").await.unwrap().is_none()); } +#[rstest] #[tokio::test] async fn approle_tokens_expire_after_the_vault_lease() { let server: MockServer = MockServer::start().await; @@ -246,6 +259,7 @@ async fn approle_tokens_expire_after_the_vault_lease() { assert!(manager.async_read_secret("second").await.unwrap().is_some()); } +#[rstest] #[tokio::test] async fn tls_login_posts_the_role_and_uses_the_client_identity() { let server: MockServer = MockServer::start().await; @@ -343,21 +357,29 @@ async fn tls_login_posts_the_role_and_uses_the_client_identity() { assert!(login_bodies.contains(&json!({}))); } -#[rstest::rstest] -#[case::missing(404, json!({"errors": ["missing"]}), 0)] -#[case::malformed(200, json!({"data": "invalid"}), 1)] -#[case::missing_key(200, json!({}), 0)] -#[case::non_string(200, json!({"key": 1}), 2)] +#[derive(Clone, Copy)] +enum ExpectedRead { + Missing, + Malformed, + NonString, +} + +#[rstest] +#[case::missing(404, json!({"errors": ["missing"]}), ExpectedRead::Missing)] +#[case::malformed(200, json!({"data": "invalid"}), ExpectedRead::Malformed)] +#[case::missing_key(200, json!({}), ExpectedRead::Missing)] +#[case::non_string(200, json!({"key": 1}), ExpectedRead::NonString)] #[tokio::test] async fn read_responses_distinguish_absence_and_malformed_payloads( + token_values: Vec<(&str, &str)>, #[case] status: u16, #[case] body: serde_json::Value, - #[case] expected: u8, + #[case] expected: ExpectedRead, ) { let server: MockServer = MockServer::start().await; Mock::given(method("GET")) .respond_with(ResponseTemplate::new(status).set_body_json( - if status == 200 && expected != 1 { + if status == 200 && !matches!(expected, ExpectedRead::Malformed) { read_response(body) } else { body @@ -366,20 +388,19 @@ async fn read_responses_distinguish_absence_and_malformed_payloads( .expect(1) .mount(&server) .await; - let result: Result, Error> = - manager(&server, &[("HCP_VAULT_TOKEN", "token")]) - .async_read_secret("name") - .await; + let result: Result, Error> = manager(&server, &token_values) + .async_read_secret("name") + .await; match expected { - 0 => assert!(result.unwrap().is_none()), - 1 => assert!(matches!(result, Err(Error::MalformedPayload))), - 2 => assert!(matches!(result, Err(Error::NonStringValue))), - _ => unreachable!(), + ExpectedRead::Missing => assert!(result.unwrap().is_none()), + ExpectedRead::Malformed => assert!(matches!(result, Err(Error::MalformedPayload))), + ExpectedRead::NonString => assert!(matches!(result, Err(Error::NonStringValue))), } } +#[rstest] #[tokio::test] -async fn write_and_delete_invalidate_the_read_cache() { +async fn write_and_delete_invalidate_the_read_cache(token_values: Vec<(&str, &str)>) { let server: MockServer = MockServer::start().await; Mock::given(method("GET")) .and(path("/v1/secret/data/name")) @@ -418,7 +439,7 @@ async fn write_and_delete_invalidate_the_read_cache() { .expect(1) .mount(&server) .await; - let manager: HashicorpVault = manager(&server, &[("HCP_VAULT_TOKEN", "token")]); + let manager: HashicorpVault = manager(&server, &token_values); assert!(manager.async_read_secret("name").await.unwrap().is_some()); assert!( @@ -431,6 +452,313 @@ async fn write_and_delete_invalidate_the_read_cache() { manager.async_delete_secret("name").await.unwrap(); } +#[rstest] +#[tokio::test] +async fn base_manager_context_overrides_vault_location_and_data_key( + token_values: Vec<(&str, &str)>, +) { + let server: MockServer = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/v1/alternate/data/managed/name")) + .respond_with( + ResponseTemplate::new(200).set_body_json(read_response(json!({"api_token": "value"}))), + ) + .expect(2) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/v1/alternate/data/managed/name")) + .and(body_json(json!({ + "data": {"api_token": "updated", "description": "Managed key"} + }))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "data": { + "created_time": "", + "deletion_time": "", + "custom_metadata": null, + "destroyed": false, + "version": 2 + }, + "lease_id": "", + "lease_duration": 0, + "renewable": false, + "request_id": "", + "warnings": null, + "wrap_info": null + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("DELETE")) + .and(path("/v1/alternate/data/managed/name")) + .respond_with(ResponseTemplate::new(204)) + .expect(1) + .mount(&server) + .await; + let manager: HashicorpVault = manager(&server, &token_values); + let operation = SecretOperationContext::Hashicorp(HashicorpOperationContext { + mount: Some(" /alternate/ ".to_owned()), + path_prefix: Some(" /managed/ ".to_owned()), + data_key: Some("api_token".to_owned()), + ..HashicorpOperationContext::default() + }); + let write_context = SecretWriteContext { + description: Some("Managed key".to_owned()), + operation: operation.clone(), + ..SecretWriteContext::default() + }; + + assert_eq!( + BaseSecretManager::async_read_secret(&manager, "name", &operation) + .await + .unwrap() + .unwrap() + .expose(), + "value" + ); + BaseSecretManager::async_write_secret( + &manager, + "name", + &SecretValue::new("updated"), + &write_context, + ) + .await + .unwrap(); + assert!( + BaseSecretManager::async_read_secret(&manager, "name", &operation) + .await + .unwrap() + .is_some() + ); + BaseSecretManager::async_delete_secret(&manager, "name", None, &operation) + .await + .unwrap(); +} + +#[rstest] +#[tokio::test] +async fn reads_cache_each_data_key_for_the_same_vault_path(token_values: Vec<(&str, &str)>) { + let server: MockServer = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/v1/secret/data/name")) + .respond_with( + ResponseTemplate::new(200).set_body_json(read_response(json!({ + "key": "primary", + "alternate": "secondary" + }))), + ) + .expect(2) + .mount(&server) + .await; + let manager: HashicorpVault = manager(&server, &token_values); + let alternate = SecretOperationContext::Hashicorp(HashicorpOperationContext { + data_key: Some("alternate".to_owned()), + ..HashicorpOperationContext::default() + }); + + assert_eq!( + manager + .async_read_secret("name") + .await + .unwrap() + .unwrap() + .expose(), + "primary" + ); + assert_eq!( + BaseSecretManager::async_read_secret(&manager, "name", &alternate) + .await + .unwrap() + .unwrap() + .expose(), + "secondary" + ); + assert_eq!( + manager + .async_read_secret("name") + .await + .unwrap() + .unwrap() + .expose(), + "primary" + ); +} + +#[rstest] +#[tokio::test] +async fn base_manager_context_timeout_limits_vault_io(token_values: Vec<(&str, &str)>) { + let server: MockServer = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/v1/secret/data/name")) + .respond_with( + ResponseTemplate::new(200) + .set_delay(Duration::from_millis(100)) + .set_body_json(read_response(json!({"key": "value"}))), + ) + .expect(1) + .mount(&server) + .await; + let manager: HashicorpVault = manager(&server, &token_values); + let context = SecretOperationContext::Hashicorp(HashicorpOperationContext { + timeout: Some(Duration::from_millis(10)), + ..HashicorpOperationContext::default() + }); + + assert!(matches!( + BaseSecretManager::async_read_secret(&manager, "name", &context).await, + Err(Error::Timeout) + )); +} + +#[rstest] +#[case::aws(SecretOperationContext::Aws(AwsOperationContext::default()))] +#[case::cyberark(SecretOperationContext::Cyberark(CyberarkOperationContext::default()))] +#[tokio::test] +async fn foreign_contexts_cannot_access_vault_secrets( + token_values: Vec<(&str, &str)>, + #[case] context: SecretOperationContext, + #[values(false, true)] cached: bool, +) { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/v1/secret/data/name")) + .respond_with( + ResponseTemplate::new(200).set_body_json(read_response(json!({"key": "value"}))), + ) + .expect(u64::from(cached)) + .mount(&server) + .await; + let manager = manager(&server, &token_values); + if cached { + assert!(manager.async_read_secret("name").await.unwrap().is_some()); + } + assert!(matches!( + BaseSecretManager::async_read_secret(&manager, "name", &context).await, + Err(Error::InvalidOperationContext) + )); + assert!(matches!( + BaseSecretManager::async_write_secret( + &manager, + "name", + &SecretValue::new("replacement"), + &SecretWriteContext { + operation: context.clone(), + ..SecretWriteContext::default() + }, + ) + .await, + Err(Error::InvalidOperationContext) + )); + assert!(matches!( + BaseSecretManager::async_delete_secret(&manager, "name", None, &context).await, + Err(Error::InvalidOperationContext) + )); + assert!(matches!( + manager + .async_rotate_secret_with_context( + "name", + "new", + &SecretValue::new("replacement"), + &context + ) + .await, + Err(Error::InvalidOperationContext) + )); + assert_eq!( + server.received_requests().await.unwrap().len(), + usize::from(cached) + ); +} + +#[rstest] +#[tokio::test] +async fn rotation_applies_timeout_to_each_request(token_values: Vec<(&str, &str)>) { + let server = MockServer::start().await; + let timeout = Duration::from_secs(1); + let delay = timeout / 2; + Mock::given(method("GET")) + .and(path("/v1/alternate/data/managed/current")) + .respond_with( + ResponseTemplate::new(200) + .set_delay(delay) + .set_body_json(read_response(json!({"api_token": "original"}))), + ) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/v1/alternate/data/managed/new")) + .and(body_json(json!({ + "data": {"api_token": "replacement", "description": "Rotated from current"} + }))) + .respond_with( + ResponseTemplate::new(200) + .set_delay(delay) + .set_body_json(json!({ + "data": { + "created_time": "", + "deletion_time": "", + "custom_metadata": null, + "destroyed": false, + "version": 1 + }, + "lease_id": "", + "lease_duration": 0, + "renewable": false, + "request_id": "", + "warnings": null, + "wrap_info": null + })), + ) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/alternate/data/managed/new")) + .respond_with( + ResponseTemplate::new(200) + .set_delay(delay) + .set_body_json(read_response(json!({"api_token": "replacement"}))), + ) + .mount(&server) + .await; + Mock::given(method("DELETE")) + .and(path("/v1/alternate/data/managed/current")) + .respond_with(ResponseTemplate::new(204).set_delay(delay)) + .mount(&server) + .await; + let manager = manager(&server, &token_values); + let context = SecretOperationContext::Hashicorp(HashicorpOperationContext { + timeout: Some(timeout), + mount: Some("alternate".to_owned()), + path_prefix: Some("managed".to_owned()), + data_key: Some("api_token".to_owned()), + }); + + manager + .async_rotate_secret_with_context( + "current", + "new", + &SecretValue::new("replacement"), + &context, + ) + .await + .unwrap(); + let requests = server.received_requests().await.unwrap(); + let operations: Vec<_> = requests + .iter() + .map(|request| (request.method.as_str(), request.url.path())) + .collect(); + assert_eq!( + operations, + [ + ("GET", "/v1/alternate/data/managed/current"), + ("POST", "/v1/alternate/data/managed/new"), + ("GET", "/v1/alternate/data/managed/new"), + ("DELETE", "/v1/alternate/data/managed/current"), + ] + ); +} + +#[rstest] #[tokio::test] async fn no_auth_and_invalid_names_fail_without_requests() { let server: MockServer = MockServer::start().await; @@ -447,6 +775,7 @@ async fn no_auth_and_invalid_names_fail_without_requests() { assert!(server.received_requests().await.unwrap().is_empty()); } +#[rstest] #[tokio::test] async fn debug_output_redacts_authentication_values() { let server: MockServer = MockServer::start().await; @@ -468,14 +797,18 @@ struct ParityCase { secret_name: String, } -#[test] -fn configuration_matches_python_parity_fixture() { - let cases: Vec = serde_json::from_str(include_str!(concat!( +#[fixture] +fn parity_cases() -> Vec { + serde_json::from_str(include_str!(concat!( env!("CARGO_MANIFEST_DIR"), "/../../../tests/test_litellm/secret_managers/hashicorp_vault_parity.json" ))) - .unwrap(); - for case in cases { + .unwrap() +} + +#[rstest] +fn configuration_matches_python_parity_fixture(parity_cases: Vec) { + for case in parity_cases { let values: HashMap = case.env.clone(); let environment: Arc = Arc::new(move |name: &str| values.get(name).cloned()); @@ -521,6 +854,7 @@ fn configuration_matches_python_parity_fixture() { } } +#[rstest] #[tokio::test] #[ignore] async fn live_vault_round_trip() { diff --git a/litellm-rust/crates/secrets-types/src/base_secret_manager.rs b/litellm-rust/crates/secrets-types/src/base_secret_manager.rs index d71bce64221..e8aed9f280e 100644 --- a/litellm-rust/crates/secrets-types/src/base_secret_manager.rs +++ b/litellm-rust/crates/secrets-types/src/base_secret_manager.rs @@ -1,4 +1,4 @@ -use crate::{Error, SecretValue}; +use crate::{Error, SecretOperationContext, SecretValue, SecretWriteContext}; pub fn validate_secret_name(name: &str) -> Result<(), Error> { if name.split('/').any(|segment| segment == "..") @@ -20,17 +20,22 @@ pub trait BaseSecretManager { type WriteResponse; type DeleteResponse; - async fn async_read_secret(&self, name: &str) -> Result, Self::Error>; + async fn async_read_secret( + &self, + name: &str, + context: &SecretOperationContext, + ) -> Result, Self::Error>; async fn async_write_secret( &self, name: &str, value: &SecretValue, - description: Option<&str>, + context: &SecretWriteContext, ) -> Result; async fn async_delete_secret( &self, name: &str, - recovery_window_in_days: i64, + recovery_window_in_days: Option, + context: &SecretOperationContext, ) -> Result; } @@ -39,20 +44,31 @@ pub async fn async_rotate_secret( current_name: &str, new_name: &str, value: &SecretValue, + context: &SecretOperationContext, ) -> Result { - if manager.async_read_secret(current_name).await?.is_none() { + if manager + .async_read_secret(current_name, context) + .await? + .is_none() + { return Err(Error::CurrentSecretMissing.into()); } let response = manager .async_write_secret( new_name, value, - Some(&format!("Rotated from {current_name}")), + &SecretWriteContext::rotated_from(current_name, context.clone()), ) .await?; - if manager.async_read_secret(new_name).await?.is_none() { + if manager + .async_read_secret(new_name, context) + .await? + .is_none() + { return Err(Error::NewSecretMissing.into()); } - manager.async_delete_secret(current_name, 7).await?; + manager + .async_delete_secret(current_name, Some(7), context) + .await?; Ok(response) } diff --git a/litellm-rust/crates/secrets-types/src/config.rs b/litellm-rust/crates/secrets-types/src/config.rs index 36d319311a3..44acf512224 100644 --- a/litellm-rust/crates/secrets-types/src/config.rs +++ b/litellm-rust/crates/secrets-types/src/config.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use crate::SecretValue; -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] pub enum KeyManagementSystem { GoogleKms, @@ -18,7 +18,7 @@ pub enum KeyManagementSystem { Custom, } -#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] pub enum AccessMode { #[default] @@ -33,7 +33,7 @@ impl AccessMode { } } -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Hash)] #[serde(default)] pub struct KeyManagementSettings { pub hosted_keys: Option>, diff --git a/litellm-rust/crates/secrets-types/src/context.rs b/litellm-rust/crates/secrets-types/src/context.rs new file mode 100644 index 00000000000..126815ab32b --- /dev/null +++ b/litellm-rust/crates/secrets-types/src/context.rs @@ -0,0 +1,65 @@ +use std::{collections::BTreeMap, time::Duration}; + +use crate::SecretValue; + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub enum SecretOperationContext { + #[default] + Default, + Aws(AwsOperationContext), + Hashicorp(HashicorpOperationContext), + Cyberark(CyberarkOperationContext), +} + +impl SecretOperationContext { + pub fn timeout(&self) -> Option { + match self { + Self::Default => None, + Self::Aws(context) => context.timeout, + Self::Hashicorp(context) => context.timeout, + Self::Cyberark(context) => context.timeout, + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct AwsOperationContext { + pub timeout: Option, + pub region_name: Option, + pub role_name: Option, + pub session_name: Option, + pub external_id: Option, + pub profile_name: Option, + pub web_identity_token: Option, + pub sts_endpoint: Option, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct HashicorpOperationContext { + pub timeout: Option, + pub mount: Option, + pub path_prefix: Option, + pub data_key: Option, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct CyberarkOperationContext { + pub timeout: Option, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct SecretWriteContext { + pub description: Option, + pub tags: BTreeMap, + pub operation: SecretOperationContext, +} + +impl SecretWriteContext { + pub fn rotated_from(current_name: &str, operation: SecretOperationContext) -> Self { + Self { + description: Some(format!("Rotated from {current_name}")), + tags: BTreeMap::new(), + operation, + } + } +} diff --git a/litellm-rust/crates/secrets-types/src/lib.rs b/litellm-rust/crates/secrets-types/src/lib.rs index 0823ed13c06..d330fc7bbea 100644 --- a/litellm-rust/crates/secrets-types/src/lib.rs +++ b/litellm-rust/crates/secrets-types/src/lib.rs @@ -2,11 +2,16 @@ mod base_secret_manager; mod config; +mod context; mod error; mod value; pub use base_secret_manager::{BaseSecretManager, async_rotate_secret, validate_secret_name}; pub use config::{AccessMode, KeyManagementSettings, KeyManagementSystem}; +pub use context::{ + AwsOperationContext, CyberarkOperationContext, HashicorpOperationContext, + SecretOperationContext, SecretWriteContext, +}; pub use error::Error; pub use litellm_auth_types::SecretValue; pub use value::Secret; diff --git a/litellm-rust/crates/secrets-types/tests/config.rs b/litellm-rust/crates/secrets-types/tests/config.rs index 4a5f17bc68a..45fd9d0ee03 100644 --- a/litellm-rust/crates/secrets-types/tests/config.rs +++ b/litellm-rust/crates/secrets-types/tests/config.rs @@ -1,15 +1,20 @@ use litellm_secrets_types::{ AccessMode, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue, }; +use rstest::{fixture, rstest}; use serde_json::json; -#[test] -fn config_preserves_defaults_nulls_and_serialized_names() { - let empty: KeyManagementSettings = serde_json::from_value(json!({})).unwrap(); - assert_eq!(empty, KeyManagementSettings::default()); - assert_eq!(empty.access_mode, AccessMode::ReadOnly); - assert_eq!(empty.store_virtual_keys, Some(false)); - assert_eq!(empty.prefix_for_stored_virtual_keys, "litellm/"); +#[fixture] +fn default_settings() -> KeyManagementSettings { + serde_json::from_value(json!({})).unwrap() +} + +#[rstest] +fn config_preserves_defaults_nulls_and_serialized_names(default_settings: KeyManagementSettings) { + assert_eq!(default_settings, KeyManagementSettings::default()); + assert_eq!(default_settings.access_mode, AccessMode::ReadOnly); + assert_eq!(default_settings.store_virtual_keys, Some(false)); + assert_eq!(default_settings.prefix_for_stored_virtual_keys, "litellm/"); let configured: KeyManagementSettings = serde_json::from_value(json!({ "hosted_keys": [], "store_virtual_keys": null, "access_mode": "write_only", "aws_web_identity_token": "private-token", "aws_external_id": "private-id", @@ -29,7 +34,7 @@ fn config_preserves_defaults_nulls_and_serialized_names() { ); } -#[rstest::rstest] +#[rstest] #[case::aws_kms("aws_kms", KeyManagementSystem::AwsKms)] #[case::aws_secret_manager("aws_secret_manager", KeyManagementSystem::AwsSecretManager)] #[case::google_kms("google_kms", KeyManagementSystem::GoogleKms)] @@ -50,11 +55,32 @@ fn key_management_system_serialization_round_trips( assert_eq!(serde_json::to_value(system).unwrap(), name); } -#[test] -fn secret_debug_never_exposes_values() { - assert!( - !format!("{:?}", Secret::String(SecretValue::new("sensitive-value"))) - .contains("sensitive-value") - ); - assert!(!format!("{:?}", Secret::Bool(true)).contains("true")); +#[rstest] +#[case::read_only(AccessMode::ReadOnly, true)] +#[case::write_only(AccessMode::WriteOnly, false)] +#[case::read_and_write(AccessMode::ReadAndWrite, true)] +fn access_mode_reports_readability(#[case] mode: AccessMode, #[case] expected: bool) { + assert_eq!(mode.readable(), expected); +} + +#[rstest] +#[case::string(json!("value"), Secret::String(SecretValue::new("value")))] +#[case::boolean(json!(true), Secret::Bool(true))] +#[case::number(json!(7), Secret::Json(json!(7)))] +#[case::null(json!(null), Secret::Json(json!(null)))] +#[case::array(json!(["value"]), Secret::Json(json!(["value"])))] +#[case::object(json!({"key": "value"}), Secret::Json(json!({"key": "value"})))] +fn secret_conversion_preserves_json_types( + #[case] value: serde_json::Value, + #[case] expected: Secret, +) { + assert_eq!(Secret::from_json(value), expected); +} + +#[rstest] +#[case::string(Secret::String(SecretValue::new("sensitive-value")), "sensitive-value")] +#[case::boolean(Secret::Bool(true), "true")] +#[case::json(Secret::Json(json!({"private": "value"})), "private")] +fn secret_debug_never_exposes_values(#[case] secret: Secret, #[case] sensitive: &str) { + assert!(!format!("{secret:?}").contains(sensitive)); } diff --git a/litellm-rust/crates/secrets-types/tests/context.rs b/litellm-rust/crates/secrets-types/tests/context.rs new file mode 100644 index 00000000000..7c9e91b3a9e --- /dev/null +++ b/litellm-rust/crates/secrets-types/tests/context.rs @@ -0,0 +1,110 @@ +use std::{collections::BTreeMap, time::Duration}; + +use litellm_secrets_types::{ + AwsOperationContext, CyberarkOperationContext, HashicorpOperationContext, + SecretOperationContext, SecretValue, SecretWriteContext, +}; +use rstest::{fixture, rstest}; + +#[fixture] +fn timeout() -> Duration { + Duration::from_secs(30) +} + +#[fixture] +fn aws_context(timeout: Duration) -> SecretOperationContext { + SecretOperationContext::Aws(AwsOperationContext { + timeout: Some(timeout), + region_name: Some("us-west-2".into()), + role_name: Some("role".into()), + external_id: Some(SecretValue::new("external-id")), + web_identity_token: Some(SecretValue::new("web-identity-token")), + ..AwsOperationContext::default() + }) +} + +#[rstest] +fn operation_context_preserves_backend_specific_values_and_redacts_secrets( + aws_context: SecretOperationContext, + timeout: Duration, +) { + assert_eq!(aws_context.timeout(), Some(timeout)); + assert_eq!( + aws_context, + SecretOperationContext::Aws(AwsOperationContext { + timeout: Some(timeout), + region_name: Some("us-west-2".into()), + role_name: Some("role".into()), + external_id: Some(SecretValue::new("external-id")), + web_identity_token: Some(SecretValue::new("web-identity-token")), + ..AwsOperationContext::default() + }) + ); + let debug = format!("{aws_context:?}"); + assert!(!debug.contains("external-id")); + assert!(!debug.contains("web-identity-token")); +} + +#[rstest] +#[case::default(SecretOperationContext::Default, None)] +#[case::aws( + SecretOperationContext::Aws(AwsOperationContext { + timeout: Some(Duration::from_secs(1)), + ..AwsOperationContext::default() + }), + Some(Duration::from_secs(1)) +)] +#[case::hashicorp( + SecretOperationContext::Hashicorp(HashicorpOperationContext { + timeout: Some(Duration::from_secs(2)), + ..HashicorpOperationContext::default() + }), + Some(Duration::from_secs(2)) +)] +#[case::cyberark( + SecretOperationContext::Cyberark(CyberarkOperationContext { + timeout: Some(Duration::from_secs(3)), + }), + Some(Duration::from_secs(3)) +)] +fn operation_context_reports_each_backend_timeout( + #[case] context: SecretOperationContext, + #[case] expected: Option, +) { + assert_eq!(context.timeout(), expected); +} + +#[rstest] +fn write_context_keeps_tags_separate_from_the_operation_context() { + let context = SecretWriteContext { + description: Some("Managed virtual key".into()), + tags: BTreeMap::from([("team".into(), "team-id".into())]), + operation: SecretOperationContext::Hashicorp(HashicorpOperationContext { + mount: Some("secret".into()), + path_prefix: Some("teams".into()), + data_key: Some("api_key".into()), + ..HashicorpOperationContext::default() + }), + }; + + assert_eq!(context.tags.get("team"), Some(&"team-id".into())); + assert_eq!(context.operation.timeout(), None); + assert_eq!( + context.operation, + SecretOperationContext::Hashicorp(HashicorpOperationContext { + mount: Some("secret".into()), + path_prefix: Some("teams".into()), + data_key: Some("api_key".into()), + ..HashicorpOperationContext::default() + }) + ); +} + +#[rstest] +fn rotation_write_context_preserves_the_operation_context(aws_context: SecretOperationContext) { + let context = SecretWriteContext::rotated_from("current", aws_context.clone()); + + assert_eq!(context.description.as_deref(), Some("Rotated from current")); + assert!(context.tags.is_empty()); + assert_eq!(context.operation, aws_context); +} diff --git a/litellm-rust/crates/secrets-types/tests/rotation.rs b/litellm-rust/crates/secrets-types/tests/rotation.rs index 48a5304ece8..1bc5e0a9eeb 100644 --- a/litellm-rust/crates/secrets-types/tests/rotation.rs +++ b/litellm-rust/crates/secrets-types/tests/rotation.rs @@ -1,12 +1,15 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use litellm_secrets_types::{ - BaseSecretManager, Error, SecretValue, async_rotate_secret, validate_secret_name, + BaseSecretManager, Error, HashicorpOperationContext, SecretOperationContext, SecretValue, + SecretWriteContext, async_rotate_secret, validate_secret_name, }; +use rstest::{fixture, rstest}; struct Manager { step: AtomicUsize, absent_at: Option, + operation: SecretOperationContext, } impl BaseSecretManager for Manager { @@ -14,7 +17,12 @@ impl BaseSecretManager for Manager { type WriteResponse = &'static str; type DeleteResponse = (); - async fn async_read_secret(&self, name: &str) -> Result, Error> { + async fn async_read_secret( + &self, + name: &str, + context: &SecretOperationContext, + ) -> Result, Error> { + assert_eq!(context, &self.operation); let step = self.step.fetch_add(1, Ordering::SeqCst); assert_eq!(name, if step == 0 { "old" } else { "new" }); Ok((self.absent_at != Some(step)).then(|| SecretValue::new("value"))) @@ -24,35 +32,54 @@ impl BaseSecretManager for Manager { &self, name: &str, value: &SecretValue, - description: Option<&str>, + context: &SecretWriteContext, ) -> Result { assert_eq!(self.step.fetch_add(1, Ordering::SeqCst), 1); assert_eq!(name, "new"); assert_eq!(value.expose(), "replacement"); - assert_eq!(description, Some("Rotated from old")); + assert_eq!(context.description.as_deref(), Some("Rotated from old")); + assert!(context.tags.is_empty()); + assert_eq!(context.operation, self.operation); Ok("provider-response") } async fn async_delete_secret( &self, name: &str, - recovery_window_in_days: i64, + recovery_window_in_days: Option, + context: &SecretOperationContext, ) -> Result<(), Error> { assert_eq!(self.step.fetch_add(1, Ordering::SeqCst), 3); assert_eq!(name, "old"); - assert_eq!(recovery_window_in_days, 7); + assert_eq!(recovery_window_in_days, Some(7)); + assert_eq!(context, &self.operation); Ok(()) } } +#[fixture] +fn replacement() -> SecretValue { + SecretValue::new("replacement") +} + +#[rstest] +#[case::default(SecretOperationContext::Default)] +#[case::backend_specific(SecretOperationContext::Hashicorp(HashicorpOperationContext { + mount: Some("alternate".to_owned()), + ..HashicorpOperationContext::default() +}))] #[tokio::test] -async fn rotation_verifies_before_deleting_and_returns_provider_response() { +async fn rotation_verifies_before_deleting_and_returns_provider_response( + replacement: SecretValue, + #[case] operation: SecretOperationContext, +) { let manager = Manager { step: AtomicUsize::new(0), absent_at: None, + operation: operation.clone(), }; assert_eq!( - async_rotate_secret(&manager, "old", "new", &SecretValue::new("replacement")) + async_rotate_secret(&manager, "old", "new", &replacement, &operation,) .await .unwrap(), "provider-response" @@ -60,11 +87,12 @@ async fn rotation_verifies_before_deleting_and_returns_provider_response() { assert_eq!(manager.step.load(Ordering::SeqCst), 4); } -#[rstest::rstest] +#[rstest] #[case::current_secret_missing(0, Error::CurrentSecretMissing, 1)] #[case::new_secret_missing(2, Error::NewSecretMissing, 3)] #[tokio::test] async fn missing_old_or_new_value_stops_rotation_before_deletion( + replacement: SecretValue, #[case] absent_at: usize, #[case] expected: Error, #[case] calls: usize, @@ -72,32 +100,57 @@ async fn missing_old_or_new_value_stops_rotation_before_deletion( let manager = Manager { step: AtomicUsize::new(0), absent_at: Some(absent_at), + operation: SecretOperationContext::default(), }; assert_eq!( - async_rotate_secret(&manager, "old", "new", &SecretValue::new("replacement")) - .await - .unwrap_err(), + async_rotate_secret( + &manager, + "old", + "new", + &replacement, + &SecretOperationContext::default(), + ) + .await + .unwrap_err(), expected ); assert_eq!(manager.step.load(Ordering::SeqCst), calls); } -#[rstest::rstest] +#[rstest] #[case::parent("..")] -#[case::parent_prefix("../x")] -#[case::parent_segment("x/../y")] -#[case::parent_suffix("x/..")] -#[case::line_feed("line\n")] -#[case::next_line("\u{85}")] -#[case::line_separator("\u{2028}")] -#[case::paragraph_separator("\u{2029}")] +#[case::parent_prefix("../../../other-app/creds")] +#[case::nested_parent_prefix("litellm/../../secret")] +#[case::parent_segment("foo/../bar")] +#[case::parent_suffix("foo/..")] +#[case::single_parent_prefix("../foo")] +#[case::line_feed("foo\nbar")] +#[case::carriage_return("foo\rbar")] +#[case::tab("foo\tbar")] +#[case::null("foo\0bar")] +#[case::delete("foo\u{7f}bar")] +#[case::next_line("foo\u{85}bar")] +#[case::line_separator("foo\u{2028}bar")] +#[case::paragraph_separator("foo\u{2029}bar")] fn names_reject_path_traversal_and_control_characters(#[case] name: &str) { assert_eq!(validate_secret_name(name), Err(Error::UnsafeSecretName)); } -#[rstest::rstest] +#[rstest] +#[case::plain_alias("plain-alias")] +#[case::key_with_digits("my-key-123")] +#[case::service_path("prod/my-service-key")] +#[case::email_path("team/user@example.com")] +#[case::colon("foo: bar")] +#[case::spaced_fragment("foo # bar")] +#[case::query("foo?evil=1")] +#[case::fragment("foo#bar")] +#[case::long_alias(&"a".repeat(500))] #[case::embedded_double_dot("release-1.0..2")] -#[case::path_separator("folder/key")] +#[case::middle_double_dot("my..key")] +#[case::leading_double_dot("..foo")] +#[case::trailing_double_dot("foo..")] +#[case::version_double_dot("v2.0..1-beta")] #[case::empty("")] #[case::three_dots("...")] fn names_allow_safe_values(#[case] name: &str) { diff --git a/litellm-rust/crates/secrets/src/error.rs b/litellm-rust/crates/secrets/src/error.rs index 1be0adc2bf5..de325ff4981 100644 --- a/litellm-rust/crates/secrets/src/error.rs +++ b/litellm-rust/crates/secrets/src/error.rs @@ -24,6 +24,8 @@ pub enum Error { OidcFile, #[error("secret cannot be converted to {expected}")] TypeMismatch { expected: &'static str }, + #[error("external secret manager failed")] + ExternalManager(#[source] Box), #[cfg(feature = "aws")] #[error(transparent)] Aws(#[from] litellm_secrets_aws::Error), diff --git a/litellm-rust/crates/secrets/src/handler.rs b/litellm-rust/crates/secrets/src/handler.rs index 5ab2caa75d4..0ae84821883 100644 --- a/litellm-rust/crates/secrets/src/handler.rs +++ b/litellm-rust/crates/secrets/src/handler.rs @@ -1,10 +1,26 @@ +use std::{future::Future, pin::Pin, sync::Arc}; + use litellm_core_utils::settings::Lookup; -use crate::{Error, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue}; +use crate::{Error, KeyManagementSettings, KeyManagementSystem, Secret}; + +#[cfg(any(feature = "aws", feature = "google"))] +use crate::SecretValue; + +pub trait ExternalSecretManager: Send + Sync { + fn system(&self) -> KeyManagementSystem; + + fn read_secret<'a>( + &'a self, + name: &'a str, + settings: &'a KeyManagementSettings, + environment: &'a (dyn Lookup + Send + Sync), + ) -> Pin, Error>> + Send + 'a>>; +} #[derive(Clone)] pub enum SecretManager { - Local, + External(Arc), #[cfg(feature = "aws")] AwsKms(crate::aws::AwsKms), #[cfg(feature = "aws")] @@ -24,7 +40,7 @@ pub enum SecretManager { impl SecretManager { pub fn system(&self) -> KeyManagementSystem { match self { - Self::Local => KeyManagementSystem::Local, + Self::External(manager) => manager.system(), #[cfg(feature = "aws")] Self::AwsKms(_) => KeyManagementSystem::AwsKms, #[cfg(feature = "aws")] @@ -50,10 +66,11 @@ pub async fn get_secret_from_manager( environment: &(dyn Lookup + Send + Sync), ) -> Result, Error> { match client { - SecretManager::Local => Ok(environment - .get(secret_name) - .map(SecretValue::new) - .map(Secret::String)), + SecretManager::External(manager) => { + manager + .read_secret(secret_name, _settings, environment) + .await + } #[cfg(feature = "aws")] SecretManager::AwsKms(client) => { let ciphertext = environment @@ -97,10 +114,9 @@ pub async fn get_secret_from_manager( .map(|value| value.map(Secret::String)) .map_err(Error::from), #[cfg(feature = "azure")] - SecretManager::AzureKeyVault(client) => client - .get_secret_from_azure_key_vault(secret_name) - .await - .map_err(Error::from), + SecretManager::AzureKeyVault(client) => { + client.get_secret(secret_name).await.map_err(Error::from) + } #[cfg(feature = "cyberark")] SecretManager::Cyberark(client) => client .async_read_secret(secret_name) diff --git a/litellm-rust/crates/secrets/src/lib.rs b/litellm-rust/crates/secrets/src/lib.rs index 1acb5269e66..58aba8494fd 100644 --- a/litellm-rust/crates/secrets/src/lib.rs +++ b/litellm-rust/crates/secrets/src/lib.rs @@ -7,7 +7,7 @@ mod resolver; mod state; pub use error::Error; -pub use handler::{SecretManager, get_secret_from_manager}; +pub use handler::{ExternalSecretManager, SecretManager, get_secret_from_manager}; pub use litellm_secrets_types::{ AccessMode, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue, }; diff --git a/litellm-rust/crates/secrets/src/resolver.rs b/litellm-rust/crates/secrets/src/resolver.rs index 89439893852..597ca11b171 100644 --- a/litellm-rust/crates/secrets/src/resolver.rs +++ b/litellm-rust/crates/secrets/src/resolver.rs @@ -72,6 +72,7 @@ impl SecretResolver { Ok(value) => Ok(value .or_else(|| self.environment_secret(name)) .or(default_value)), + Err(error @ Error::ExternalManager(_)) => Err(error), Err(error) => match self.failure_policy { FailurePolicy::Propagate => Err(error), FailurePolicy::EnvironmentFallback => self diff --git a/litellm-rust/crates/secrets/tests/resolution.rs b/litellm-rust/crates/secrets/tests/resolution.rs index 3a826092d72..98b34a2c72e 100644 --- a/litellm-rust/crates/secrets/tests/resolution.rs +++ b/litellm-rust/crates/secrets/tests/resolution.rs @@ -1,19 +1,14 @@ use std::sync::Arc; use litellm_secrets::{ - Error, KeyManagementSettings, OidcResolver, Secret, SecretManager, SecretManagerState, - SecretResolver, SecretValue, secret_manager_would_be_consulted, + Error, OidcResolver, Secret, SecretManagerState, SecretResolver, SecretValue, + secret_manager_would_be_consulted, }; -fn resolver(value: Option<&str>, configured: bool) -> SecretResolver { - let state = if configured { - SecretManagerState::new(SecretManager::Local, KeyManagementSettings::default()) - } else { - SecretManagerState::default() - }; +fn resolver(value: Option<&str>) -> SecretResolver { let value = value.map(str::to_owned); SecretResolver::new( - Arc::new(state), + Arc::new(SecretManagerState::default()), Arc::new(move |_: &str| value.clone()), OidcResolver::default(), ) @@ -30,9 +25,8 @@ fn resolver(value: Option<&str>, configured: bool) -> SecretResolver { async fn conversion_is_explicit_and_independent_of_manager_configuration( #[case] input: &str, #[case] boolean: Option, - #[values(false, true)] configured: bool, ) { - let resolver = resolver(Some(input), configured); + let resolver = resolver(Some(input)); assert_eq!( resolver.get_secret("key", None).await.unwrap(), Some(Secret::String(SecretValue::new(input))) @@ -62,8 +56,8 @@ async fn conversion_is_explicit_and_independent_of_manager_configuration( #[rstest::rstest] #[tokio::test] -async fn defaults_apply_only_to_absence(#[values(false, true)] configured: bool) { - let missing = resolver(None, configured); +async fn defaults_apply_only_to_absence() { + let missing = resolver(None); assert_eq!(missing.get_secret("key", None).await.unwrap(), None); assert_eq!( missing.get_secret_bool("key", Some(false)).await.unwrap(), @@ -92,7 +86,7 @@ async fn defaults_apply_only_to_absence(#[values(false, true)] configured: bool) ); } assert_eq!( - resolver(Some(""), configured) + resolver(Some("")) .get_secret_str("key", Some(SecretValue::new("default"))) .await .unwrap() @@ -103,12 +97,8 @@ async fn defaults_apply_only_to_absence(#[values(false, true)] configured: bool) } #[tokio::test] -async fn prefix_is_removed_once_and_local_manager_is_not_consulted() { - let state = SecretManagerState::new(SecretManager::Local, KeyManagementSettings::default()); - assert_eq!( - state.system(), - Some(litellm_secrets::KeyManagementSystem::Local) - ); +async fn prefix_is_removed_once_and_resolved_from_environment() { + let state = SecretManagerState::default(); assert!(!secret_manager_would_be_consulted( &state, "os.environ/os.environ/KEY" @@ -131,7 +121,7 @@ async fn prefix_is_removed_once_and_local_manager_is_not_consulted() { #[tokio::test] async fn resolver_future_can_run_on_a_tokio_worker() { - let resolver = resolver(Some("worker-value"), false); + let resolver = resolver(Some("worker-value")); let result = tokio::spawn(async move { resolver.get_secret_str("KEY", None).await }) .await .unwrap() @@ -142,7 +132,9 @@ async fn resolver_future_can_run_on_a_tokio_worker() { #[cfg(feature = "aws")] mod aws { use super::*; - use litellm_secrets::{AccessMode, FailurePolicy, aws::AwsSecretsManagerV2}; + use litellm_secrets::{ + AccessMode, FailurePolicy, KeyManagementSettings, SecretManager, aws::AwsSecretsManagerV2, + }; use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method}; fn state(server: &MockServer, settings: KeyManagementSettings) -> SecretManagerState { @@ -319,7 +311,9 @@ mod aws { #[case::failure(503)] #[tokio::test] async fn google_resolver_distinguishes_absence_from_failure(#[case] status: u16) { - use litellm_secrets::{FailurePolicy, google::GoogleSecretManager}; + use litellm_secrets::{ + FailurePolicy, KeyManagementSettings, SecretManager, google::GoogleSecretManager, + }; use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method}; let server = MockServer::start().await; Mock::given(method("GET")) diff --git a/litellm-rust/crates/token-counter-fast/src/lib.rs b/litellm-rust/crates/token-counter-fast/src/lib.rs index ce91af642ea..157ee847658 100644 --- a/litellm-rust/crates/token-counter-fast/src/lib.rs +++ b/litellm-rust/crates/token-counter-fast/src/lib.rs @@ -8,6 +8,8 @@ mod scanner; mod tiktoken; mod unicode_classes; +use std::sync::Arc; + use byte_level::ByteLevelCounter; use scanner::{SplitPattern, TiktokenCounter}; @@ -15,22 +17,29 @@ pub use error::Error; enum Encoder { HuggingFace { - tokenizer: Box, + tokenizer: Arc, byte_level: Option, }, Tiktoken(TiktokenCounter), } +/// A count-only tokenizer. Its model tables are immutable, so one built from an already +/// loaded model (`from_shared`, `from_*_pairs`) adds only the count-specific tables. pub struct FastTokenizer(Encoder); impl FastTokenizer { pub fn from_json(json: &str) -> Result { let tokenizer = json.parse::().map_err(Error::Load)?; + Ok(Self::from_shared(Arc::new(tokenizer))) + } + + /// Counts with a Hugging Face model another codec already holds; nothing is re-parsed. + pub fn from_shared(tokenizer: Arc) -> Self { let byte_level = ByteLevelCounter::detect(&tokenizer); - Ok(Self(Encoder::HuggingFace { - tokenizer: Box::new(tokenizer), + Self(Encoder::HuggingFace { + tokenizer, byte_level, - })) + }) } pub fn from_cl100k_ranks(ranks: &str) -> Result { @@ -41,12 +50,36 @@ impl FastTokenizer { Self::from_ranks(SplitPattern::O200k, ranks) } + /// `cl100k_base` from ranks another loader already parsed. + pub fn from_cl100k_pairs<'a>( + pairs: impl IntoIterator, + ) -> Result { + Self::from_pairs(SplitPattern::Cl100k, pairs) + } + + /// `o200k_base` (and `o200k_harmony`, whose ordinary tokens are the same) from ranks + /// another loader already parsed. + pub fn from_o200k_pairs<'a>( + pairs: impl IntoIterator, + ) -> Result { + Self::from_pairs(SplitPattern::O200k, pairs) + } + fn from_ranks(split: SplitPattern, ranks: &str) -> Result { TiktokenCounter::from_ranks(split, ranks) .map(Encoder::Tiktoken) .map(Self) } + fn from_pairs<'a>( + split: SplitPattern, + pairs: impl IntoIterator, + ) -> Result { + TiktokenCounter::from_pairs(split, pairs) + .map(Encoder::Tiktoken) + .map(Self) + } + pub fn count_tokens(&self, text: &str) -> Result { match &self.0 { Encoder::Tiktoken(counter) => Ok(counter.count(text)), diff --git a/litellm-rust/crates/token-counter-fast/src/scanner.rs b/litellm-rust/crates/token-counter-fast/src/scanner.rs index c2c3057aeeb..882ea91db81 100644 --- a/litellm-rust/crates/token-counter-fast/src/scanner.rs +++ b/litellm-rust/crates/token-counter-fast/src/scanner.rs @@ -39,8 +39,19 @@ pub(super) struct TiktokenCounter { impl TiktokenCounter { pub(super) fn from_ranks(split: SplitPattern, rank_file: &str) -> Result { + Self::new(split, MergeRanks::parse(rank_file)?) + } + + pub(super) fn from_pairs<'a>( + split: SplitPattern, + pairs: impl IntoIterator, + ) -> Result { + Self::new(split, MergeRanks::from_pairs(pairs)?) + } + + fn new(split: SplitPattern, ranks: MergeRanks) -> Result { Ok(Self { - ranks: MergeRanks::parse(rank_file)?, + ranks, piece_len: split.piece_len(), unicode_classes: UnicodeClasses::get().ok_or(Error::UnicodeClasses)?, }) diff --git a/litellm-rust/crates/token-counter-fast/src/tiktoken.rs b/litellm-rust/crates/token-counter-fast/src/tiktoken.rs index 16172b7a688..68f09b14a25 100644 --- a/litellm-rust/crates/token-counter-fast/src/tiktoken.rs +++ b/litellm-rust/crates/token-counter-fast/src/tiktoken.rs @@ -22,11 +22,25 @@ pub(super) struct MergeRanks(FxHashMap, Rank>); impl MergeRanks { pub(super) fn parse(text: &str) -> Result { - let ranks = text - .lines() - .filter(|line| !line.is_empty()) - .map(parse_line) - .collect::, _>>()?; + Self::from_entries(text.lines().filter(|line| !line.is_empty()).map(parse_line)) + } + + /// The same table from ranks another loader already parsed. + pub(super) fn from_pairs<'a>( + pairs: impl IntoIterator, + ) -> Result { + Self::from_entries(pairs.into_iter().map(|(bytes, rank)| { + if rank == NO_RANK { + return Err(Error::Ranks(format!("rank {rank} is reserved"))); + } + Ok((Box::from(bytes), rank)) + })) + } + + fn from_entries( + entries: impl Iterator, Rank), Error>>, + ) -> Result { + let ranks = entries.collect::, _>>()?; if let Some(byte) = (0..=u8::MAX).find(|byte| !ranks.contains_key(&[*byte][..])) { return Err(Error::Ranks(format!("byte 0x{byte:02X} has no token"))); } diff --git a/litellm-rust/crates/token-counter-huggingface/Cargo.toml b/litellm-rust/crates/token-counter-huggingface/Cargo.toml index 6d8cb85e524..a5c2b1bb160 100644 --- a/litellm-rust/crates/token-counter-huggingface/Cargo.toml +++ b/litellm-rust/crates/token-counter-huggingface/Cargo.toml @@ -6,5 +6,6 @@ license.workspace = true repository.workspace = true [dependencies] +serde_json.workspace = true thiserror.workspace = true tokenizers.workspace = true diff --git a/litellm-rust/crates/token-counter-huggingface/src/error.rs b/litellm-rust/crates/token-counter-huggingface/src/error.rs index adc4551886f..e7f6260321b 100644 --- a/litellm-rust/crates/token-counter-huggingface/src/error.rs +++ b/litellm-rust/crates/token-counter-huggingface/src/error.rs @@ -6,4 +6,6 @@ pub enum Error { Load(#[source] tokenizers::Error), #[error("tokenization failed: {0}")] Encode(#[source] tokenizers::Error), + #[error("token decoding failed: {0}")] + Decode(#[source] tokenizers::Error), } diff --git a/litellm-rust/crates/token-counter-huggingface/src/lib.rs b/litellm-rust/crates/token-counter-huggingface/src/lib.rs index 8e05c2cca46..170a36aea05 100644 --- a/litellm-rust/crates/token-counter-huggingface/src/lib.rs +++ b/litellm-rust/crates/token-counter-huggingface/src/lib.rs @@ -2,22 +2,243 @@ mod error; -pub use error::Error; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; -pub struct HuggingFaceTokenizer(Box); +pub use error::Error; +use tokenizers::PostProcessor; +pub use tokenizers::{ + AddedToken, EncodeInput, Encoding, InputSequence, PaddingDirection, PaddingParams, + PaddingStrategy, TruncationDirection, TruncationParams, +}; + +pub fn encoding_from_json(json: &str) -> Result { + serde_json::from_str(json).map_err(|error| Error::Load(error.into())) +} + +pub fn encoding_to_json(encoding: &Encoding) -> Result { + serde_json::to_string(encoding).map_err(|error| Error::Load(error.into())) +} + +pub struct HuggingFaceTokenizer { + tokenizer: Arc, + special_token_ids: HashSet, +} impl HuggingFaceTokenizer { pub fn from_json(json: &str) -> Result { json.parse::() - .map(Box::new) - .map(Self) + .map(Self::new) .map_err(Error::Load) } + fn new(tokenizer: tokenizers::Tokenizer) -> Self { + let special_token_ids: HashSet = tokenizer + .get_added_tokens_decoder() + .into_iter() + .filter_map(|(id, token)| token.special.then_some(id)) + .collect(); + Self { + tokenizer: Arc::new(tokenizer), + special_token_ids, + } + } + + /// The parsed model, for a count-only counter to share instead of parsing it again. + pub fn shared(&self) -> Arc { + Arc::clone(&self.tokenizer) + } + pub fn count_tokens(&self, text: &str) -> Result { - self.0 + self.tokenizer .encode_fast(text, true) .map(|encoding| encoding.len()) .map_err(Error::Encode) } + + pub fn encode(&self, text: &str) -> Result, Error> { + self.tokenizer + .encode_fast(text, true) + .map(|encoding| encoding.get_ids().to_vec()) + .map_err(Error::Encode) + } + + pub fn encode_result<'a>( + &self, + input: EncodeInput<'a>, + add_special_tokens: bool, + fast: bool, + ) -> Result { + if fast { + return self + .tokenizer + .encode_fast(input, add_special_tokens) + .map_err(Error::Encode); + } + self.tokenizer + .encode_char_offsets(input, add_special_tokens) + .map_err(Error::Encode) + } + + pub fn encode_batch_result<'a>( + &self, + inputs: Vec>, + add_special_tokens: bool, + fast: bool, + ) -> Result, Error> { + if fast { + return self + .tokenizer + .encode_batch_fast(inputs, add_special_tokens) + .map_err(Error::Encode); + } + self.tokenizer + .encode_batch_char_offsets(inputs, add_special_tokens) + .map_err(Error::Encode) + } + + pub fn to_json(&self, pretty: bool) -> Result { + self.tokenizer.to_string(pretty).map_err(Error::Load) + } + + pub fn token_to_id(&self, token: &str) -> Option { + self.tokenizer.token_to_id(token) + } + + pub fn id_to_token(&self, id: u32) -> Option { + self.tokenizer.id_to_token(id) + } + + pub fn vocab(&self, with_added_tokens: bool) -> HashMap { + self.tokenizer.get_vocab(with_added_tokens) + } + + pub fn vocab_size(&self, with_added_tokens: bool) -> usize { + self.tokenizer.get_vocab_size(with_added_tokens) + } + + /// The added tokens by id, in id order. + pub fn added_tokens_decoder(&self) -> Vec<(u32, AddedToken)> { + let mut added: Vec<(u32, AddedToken)> = self + .tokenizer + .get_added_tokens_decoder() + .into_iter() + .collect(); + added.sort_unstable_by_key(|(id, _)| *id); + added + } + + pub fn padding(&self) -> Option<&PaddingParams> { + self.tokenizer.get_padding() + } + + pub fn truncation(&self) -> Option<&TruncationParams> { + self.tokenizer.get_truncation() + } + + /// How many special tokens the post-processor adds to a single sequence or a pair. + pub fn num_special_tokens_to_add(&self, is_pair: bool) -> usize { + self.tokenizer + .get_post_processor() + .map_or(0, |processor| processor.added_tokens(is_pair)) + } + + pub fn encode_special_tokens(&self) -> bool { + self.tokenizer.get_encode_special_tokens() + } + + pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result { + if !skip_special_tokens { + return self.tokenizer.decode(ids, false).map_err(Error::Decode); + } + let filtered_ids: Vec = ids + .iter() + .copied() + .filter(|id| !self.special_token_ids.contains(id)) + .collect(); + self.tokenizer + .decode(&filtered_ids, true) + .map_err(Error::Decode) + } + + pub fn name(&self) -> &str { + "huggingface" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn codecs_round_trip_and_skip_special_tokens() { + let json = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json" + )); + let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap(); + let ids = tokenizer.encode("hello").unwrap(); + + assert!(tokenizer.decode(&ids, false).unwrap().contains("")); + assert_eq!(tokenizer.decode(&ids, true).unwrap(), "hello"); + } + + #[test] + fn decode_filters_special_added_tokens() { + let json = r#"{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 1, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": {"type": "Whitespace"}, + "post_processor": null, + "decoder": null, + "model": { + "type": "WordLevel", + "vocab": {"": 0, "": 1, "hello": 2}, + "unk_token": "" + } + }"#; + let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap(); + + assert!(!tokenizer.decode(&[1, 2], true).unwrap().contains("")); + assert!(tokenizer.decode(&[1, 2], false).unwrap().contains("")); + } + + #[test] + fn vocabulary_lookups_mirror_the_tokenizers_api() { + let json = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json" + )); + let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap(); + let ids = tokenizer.encode("hello").unwrap(); + + let token = tokenizer.id_to_token(ids[0]).unwrap(); + assert_eq!(tokenizer.token_to_id(&token), Some(ids[0])); + assert_eq!(tokenizer.id_to_token(u32::MAX), None); + assert_eq!(tokenizer.vocab(true).len(), tokenizer.vocab_size(true)); + assert!(tokenizer.vocab_size(true) >= tokenizer.vocab_size(false)); + let added = tokenizer.added_tokens_decoder(); + assert!(added.windows(2).all(|pair| pair[0].0 < pair[1].0)); + assert!(added.iter().any(|(_, token)| token.special)); + assert!(tokenizer.padding().is_none()); + assert!(tokenizer.truncation().is_none()); + assert!(!tokenizer.encode_special_tokens()); + assert_eq!( + tokenizer.num_special_tokens_to_add(false), + tokenizer.encode("").unwrap().len() + ); + } } diff --git a/litellm-rust/crates/token-counter-tiktoken/Cargo.toml b/litellm-rust/crates/token-counter-tiktoken/Cargo.toml index 494a9233e69..2fb3103e0c8 100644 --- a/litellm-rust/crates/token-counter-tiktoken/Cargo.toml +++ b/litellm-rust/crates/token-counter-tiktoken/Cargo.toml @@ -6,5 +6,8 @@ license.workspace = true repository.workspace = true [dependencies] +base64.workspace = true +once_cell = "1.21.3" +rustc-hash = "2.1.3" thiserror.workspace = true tiktoken-rs.workspace = true diff --git a/litellm-rust/crates/token-counter-tiktoken/src/lib.rs b/litellm-rust/crates/token-counter-tiktoken/src/lib.rs index ecdb3946eee..f049e90a1cb 100644 --- a/litellm-rust/crates/token-counter-tiktoken/src/lib.rs +++ b/litellm-rust/crates/token-counter-tiktoken/src/lib.rs @@ -1,27 +1,125 @@ #![forbid(unsafe_code)] mod error; +mod ranks; + +use std::collections::HashSet; pub use error::UnsupportedTokenizer; +pub use ranks::{LoadError, Vocabulary}; -pub struct TiktokenTokenizer(&'static tiktoken_rs::CoreBPE); +pub struct TiktokenTokenizer { + encoder: &'static tiktoken_rs::CoreBPE, + /// Present for encodings built from a rank file; the embedded tiktoken-rs singletons + /// behind [`from_name`](Self::from_name) keep their ranks private. + vocabulary: Option<&'static Vocabulary>, + name: &'static str, +} impl TiktokenTokenizer { + /// Builds `name` from its packaged rank file (read through `load`), once per process. + /// The tokenizer reports the requested name, so `gpt2` stays `gpt2` like tiktoken does. + pub fn from_cached_ranks( + name: &str, + load: impl FnOnce(&str) -> std::io::Result, + ) -> Result { + let (loaded, name) = ranks::load(name, load)?; + Ok(Self { + encoder: &loaded.bpe, + vocabulary: Some(&loaded.vocabulary), + name, + }) + } + + /// The encodings tiktoken-rs embeds, for hosts without the packaged rank files. pub fn from_name(name: &str) -> Result { - let tokenizer = match name { - "cl100k_base" => tiktoken_rs::cl100k_base_singleton(), - "o200k_base" => tiktoken_rs::o200k_base_singleton(), - "o200k_harmony" => tiktoken_rs::o200k_harmony_singleton(), - "p50k_base" => tiktoken_rs::p50k_base_singleton(), - "p50k_edit" => tiktoken_rs::p50k_edit_singleton(), - "r50k_base" | "gpt2" => tiktoken_rs::r50k_base_singleton(), + let (encoder, name) = match name { + "cl100k_base" => (tiktoken_rs::cl100k_base_singleton(), "cl100k_base"), + "o200k_base" => (tiktoken_rs::o200k_base_singleton(), "o200k_base"), + "o200k_harmony" => (tiktoken_rs::o200k_harmony_singleton(), "o200k_harmony"), + "p50k_base" => (tiktoken_rs::p50k_base_singleton(), "p50k_base"), + "p50k_edit" => (tiktoken_rs::p50k_edit_singleton(), "p50k_edit"), + "r50k_base" => (tiktoken_rs::r50k_base_singleton(), "r50k_base"), + "gpt2" => (tiktoken_rs::r50k_base_singleton(), "gpt2"), _ => return Err(UnsupportedTokenizer(name.to_owned())), }; - Ok(Self(tokenizer)) + Ok(Self { + encoder, + vocabulary: None, + name, + }) + } + + pub fn vocabulary(&self) -> Option<&Vocabulary> { + self.vocabulary } pub fn count_tokens(&self, text: &str) -> usize { - self.0.count_ordinary(text) + self.encoder.count_ordinary(text) + } + + pub fn encode(&self, text: &str) -> Vec { + self.encoder.encode_ordinary(text) + } + + pub fn encode_special(&self, text: &str, allowed: &[String]) -> Result, String> { + let allowed = allowed.iter().map(String::as_str).collect(); + self.encoder + .encode(text, &allowed) + .map(|(ids, _)| ids) + .map_err(|error| error.to_string()) + } + + pub fn special_tokens(&self) -> HashSet { + self.encoder + .special_tokens() + .into_iter() + .map(str::to_owned) + .collect() + } + + /// tiktoken's `encode_with_unstable`: the stable prefix of `text`'s tokens and every + /// token sequence the unstable tail could still become, sorted for a stable order. + pub fn encode_with_unstable( + &self, + text: &str, + allowed: &[String], + ) -> (Vec, Vec>) { + let allowed = allowed.iter().map(String::as_str).collect(); + let (stable, completions) = self.encoder._encode_unstable_native(text, &allowed); + let mut completions: Vec> = completions.into_iter().collect(); + completions.sort_unstable(); + (stable, completions) + } + + pub fn decode_bytes(&self, ids: &[u32]) -> Result, String> { + self.encoder + .decode_bytes(ids) + .map_err(|error| error.to_string()) + } + + pub fn decode(&self, ids: &[u32]) -> Result { + self.encoder + .decode_bytes(ids) + .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) + .map_err(|error| error.to_string()) + } + + pub fn name(&self) -> &str { + self.name + } +} + +pub fn encoding_for_model(model: &str) -> Option<&'static str> { + match tiktoken_rs::tokenizer::get_tokenizer(model)? { + tiktoken_rs::tokenizer::Tokenizer::Cl100kBase => Some("cl100k_base"), + tiktoken_rs::tokenizer::Tokenizer::O200kBase => Some("o200k_base"), + tiktoken_rs::tokenizer::Tokenizer::O200kHarmony => Some("o200k_harmony"), + tiktoken_rs::tokenizer::Tokenizer::P50kBase => Some("p50k_base"), + tiktoken_rs::tokenizer::Tokenizer::P50kEdit => Some("p50k_edit"), + tiktoken_rs::tokenizer::Tokenizer::R50kBase | tiktoken_rs::tokenizer::Tokenizer::Gpt2 => { + Some("r50k_base") + } } } @@ -66,5 +164,75 @@ mod tests { panic!("unknown encoding must be rejected"); }; assert_eq!(name, "unknown-encoding"); + assert_eq!(TiktokenTokenizer::from_name("gpt2").unwrap().name(), "gpt2"); + } + + #[test] + fn codecs_round_trip_named_encodings() { + let encodings = [ + "cl100k_base", + "o200k_base", + "o200k_harmony", + "p50k_base", + "p50k_edit", + "r50k_base", + "gpt2", + ]; + let texts = ["hello world", "café 漢字 مرحبا 🙂", "line one\nline two"]; + for name in encodings { + let tokenizer = TiktokenTokenizer::from_name(name).unwrap(); + for text in texts { + assert_eq!( + tokenizer.decode(&tokenizer.encode(text)).unwrap(), + text, + "{name}: {text:?}", + ); + } + } + } + + #[test] + fn decoding_token_prefixes_replaces_incomplete_utf8() { + let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap(); + let reference = tiktoken_rs::cl100k_base_singleton(); + let ids = tokenizer.encode("🙂漢字"); + for end in 1..=ids.len() { + let bytes = reference.decode_bytes(&ids[..end]).unwrap(); + assert_eq!( + tokenizer.decode(&ids[..end]).unwrap(), + String::from_utf8_lossy(&bytes), + ); + } + assert!(tokenizer.decode(&[u32::MAX]).is_err()); + } + + #[test] + fn unstable_encoding_prefixes_stay_consistent_with_full_encoding() { + let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap(); + let text = "hello fanta"; + let (stable, completions) = tokenizer.encode_with_unstable(text, &[]); + assert!( + text.as_bytes() + .starts_with(&tokenizer.decode_bytes(&stable).unwrap()) + ); + assert!(!completions.is_empty()); + for completion in &completions { + let mut ids = stable.clone(); + ids.extend(completion); + assert!( + tokenizer + .decode_bytes(&ids) + .unwrap() + .starts_with(text.as_bytes()) + ); + } + assert!(completions.windows(2).all(|pair| pair[0] < pair[1])); + } + + #[test] + fn encoding_for_model_maps_known_models() { + assert_eq!(encoding_for_model("gpt-4o"), Some("o200k_base")); + assert_eq!(encoding_for_model("text-davinci-003"), Some("p50k_base")); + assert_eq!(encoding_for_model("unknown-model"), None); } } diff --git a/litellm-rust/crates/token-counter-tiktoken/src/ranks.rs b/litellm-rust/crates/token-counter-tiktoken/src/ranks.rs new file mode 100644 index 00000000000..1f5e5262de5 --- /dev/null +++ b/litellm-rust/crates/token-counter-tiktoken/src/ranks.rs @@ -0,0 +1,340 @@ +use base64::{Engine, engine::general_purpose::STANDARD}; +use once_cell::sync::OnceCell; +use rustc_hash::FxHashMap; +use thiserror::Error; +use tiktoken_rs::{CoreBPE, O200K_BASE_PAT_STR, Rank}; + +use crate::UnsupportedTokenizer; + +const CL100K: &str = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4"; +const O200K: &str = "fb374d419588a4632f3f557e76b4b70aebbca790"; +const P50K: &str = "ec7223a39ce59f226a68acc30dc1af2788490e15"; +const LEGACY_PATTERN: &str = + r"'(?:[sdmt]|ll|ve|re)| ?\p{L}++| ?\p{N}++| ?[^\s\p{L}\p{N}]++|\s++$|\s+(?!\S)|\s"; +const CL100K_PATTERN: &str = r"'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s"; + +static CL100K_ENCODER: OnceCell = OnceCell::new(); +static O200K_ENCODER: OnceCell = OnceCell::new(); +static HARMONY_ENCODER: OnceCell = OnceCell::new(); +static P50K_ENCODER: OnceCell = OnceCell::new(); +static EDIT_ENCODER: OnceCell = OnceCell::new(); +static R50K_ENCODER: OnceCell = OnceCell::new(); + +/// One encoding built from a rank file: the BPE engine plus the vocabulary it was built +/// from, kept because `CoreBPE` does not expose its ranks and tiktoken's Python API does +/// (`token_byte_values`, `encode_single_token`, `max_token_value`, `_special_tokens`). +pub(super) struct Loaded { + pub(super) bpe: CoreBPE, + pub(super) vocabulary: Vocabulary, +} + +/// The byte-level vocabulary of a tiktoken encoding. +pub struct Vocabulary { + ranks: FxHashMap, Rank>, + special_tokens: FxHashMap, + max_token_value: Rank, +} + +impl Vocabulary { + /// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`. + pub fn token_byte_values(&self) -> Vec> { + let mut values: Vec> = self.ranks.keys().cloned().collect(); + values.sort_unstable(); + values + } + + /// The rank of one whole token: a mergeable piece first, then a special token's text. + pub fn encode_single_token(&self, piece: &[u8]) -> Option { + if let Some(rank) = self.ranks.get(piece) { + return Some(*rank); + } + std::str::from_utf8(piece) + .ok() + .and_then(|text| self.special_tokens.get(text).copied()) + } + + pub fn max_token_value(&self) -> Rank { + self.max_token_value + } + + /// Every mergeable token with its rank, for building other tables from one parse. + pub fn ranks(&self) -> impl Iterator + '_ { + self.ranks + .iter() + .map(|(bytes, rank)| (bytes.as_slice(), *rank)) + } + + /// The special tokens with their ranks, tiktoken's `_special_tokens`. + pub fn special_tokens(&self) -> impl Iterator + '_ { + self.special_tokens + .iter() + .map(|(token, rank)| (token.as_str(), *rank)) + } + + pub fn is_special_token(&self, rank: Rank) -> bool { + self.special_tokens.values().any(|special| *special == rank) + } +} + +#[derive(Debug, Error)] +pub enum LoadError { + #[error(transparent)] + Unsupported(#[from] UnsupportedTokenizer), + #[error("failed to load tiktoken ranks: {0}")] + Ranks(String), +} + +/// Loads `name` once per process. The returned name is the one requested (`gpt2` stays +/// `gpt2`, as `tiktoken.get_encoding("gpt2").name` does), while `gpt2` and `r50k_base` share +/// one cached encoder. +pub(super) fn load( + name: &str, + load_file: impl FnOnce(&str) -> std::io::Result, +) -> Result<(&'static Loaded, &'static str), LoadError> { + let (requested, canonical, file, cache) = match name { + "cl100k_base" => ("cl100k_base", "cl100k_base", CL100K, &CL100K_ENCODER), + "o200k_base" => ("o200k_base", "o200k_base", O200K, &O200K_ENCODER), + "o200k_harmony" => ("o200k_harmony", "o200k_harmony", O200K, &HARMONY_ENCODER), + "p50k_base" => ("p50k_base", "p50k_base", P50K, &P50K_ENCODER), + "p50k_edit" => ("p50k_edit", "p50k_edit", P50K, &EDIT_ENCODER), + "r50k_base" => ("r50k_base", "r50k_base", P50K, &R50K_ENCODER), + "gpt2" => ("gpt2", "r50k_base", P50K, &R50K_ENCODER), + _ => return Err(UnsupportedTokenizer(name.to_owned()).into()), + }; + let loaded = cache.get_or_try_init(|| { + let ranks = load_file(file).map_err(|error| LoadError::Ranks(error.to_string()))?; + build(canonical, &ranks) + })?; + Ok((loaded, requested)) +} + +fn build(name: &str, ranks: &str) -> Result { + let parsed = ranks + .lines() + .map(parse_rank) + .collect::, _>>()?; + let encoder: FxHashMap<_, _> = parsed + .into_iter() + .filter(|(_, rank)| name != "r50k_base" || *rank < 50256) + .collect(); + if encoder + .values() + .collect::>() + .len() + != encoder.len() + || (0..=u8::MAX).any(|byte| !encoder.contains_key(&[byte][..])) + { + return Err(LoadError::Ranks("invalid vocabulary ranks".into())); + } + let (pattern, specials): (&str, &[(&str, Rank)]) = match name { + "cl100k_base" => ( + CL100K_PATTERN, + &[ + ("<|endoftext|>", 100257), + ("<|fim_prefix|>", 100258), + ("<|fim_middle|>", 100259), + ("<|fim_suffix|>", 100260), + ("<|endofprompt|>", 100276), + ], + ), + "o200k_base" => ( + O200K_BASE_PAT_STR, + &[("<|endoftext|>", 199999), ("<|endofprompt|>", 200018)], + ), + "o200k_harmony" => ( + O200K_BASE_PAT_STR, + &[ + ("<|startoftext|>", 199998), + ("<|endoftext|>", 199999), + ("<|reserved_200000|>", 200000), + ("<|reserved_200001|>", 200001), + ("<|return|>", 200002), + ("<|constrain|>", 200003), + ("<|reserved_200004|>", 200004), + ("<|channel|>", 200005), + ("<|start|>", 200006), + ("<|end|>", 200007), + ("<|message|>", 200008), + ("<|reserved_200009|>", 200009), + ("<|reserved_200010|>", 200010), + ("<|reserved_200011|>", 200011), + ("<|call|>", 200012), + ], + ), + "p50k_edit" => ( + LEGACY_PATTERN, + &[ + ("<|endoftext|>", 50256), + ("<|fim_prefix|>", 50281), + ("<|fim_middle|>", 50282), + ("<|fim_suffix|>", 50283), + ], + ), + _ => (LEGACY_PATTERN, &[("<|endoftext|>", 50256)]), + }; + let reserved = (200013..=201087) + .filter(|_| name == "o200k_harmony") + .map(|rank| (format!("<|reserved_{rank}|>"), rank)); + let special_tokens: FxHashMap = specials + .iter() + .map(|(token, rank)| ((*token).to_owned(), *rank)) + .chain(reserved) + .collect(); + let max_token_value = encoder + .values() + .chain(special_tokens.values()) + .copied() + .max() + .ok_or_else(|| LoadError::Ranks("empty vocabulary".into()))?; + let bpe = CoreBPE::new(encoder.clone(), special_tokens.clone(), pattern) + .map_err(|error| LoadError::Ranks(error.to_string()))?; + Ok(Loaded { + bpe, + vocabulary: Vocabulary { + ranks: encoder, + special_tokens, + max_token_value, + }, + }) +} + +fn parse_rank(line: &str) -> Result<(Vec, Rank), LoadError> { + let (token, rank) = line + .split_once(' ') + .ok_or_else(|| LoadError::Ranks("missing rank".into()))?; + let bytes = STANDARD + .decode(token) + .map_err(|error| LoadError::Ranks(error.to_string()))?; + let rank = rank + .parse() + .map_err(|error: std::num::ParseIntError| LoadError::Ranks(error.to_string()))?; + Ok((bytes, rank)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::TiktokenTokenizer; + + fn read_packaged_ranks(file: &str) -> std::io::Result { + std::fs::read_to_string( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../../litellm/litellm_core_utils/tokenizers") + .join(file), + ) + } + + #[test] + fn packaged_encodings_match_embedded_encodings_and_reuse_successful_loads() { + for name in [ + "cl100k_base", + "o200k_base", + "o200k_harmony", + "p50k_base", + "p50k_edit", + "r50k_base", + "gpt2", + ] { + if name != "gpt2" { + assert!( + TiktokenTokenizer::from_cached_ranks(name, |_| { + Err(std::io::Error::other("unreadable vocabulary")) + }) + .is_err() + ); + } + let loads = std::sync::atomic::AtomicUsize::new(0); + let barrier = std::sync::Barrier::new(4); + let encoders = std::thread::scope(|scope| { + let tasks: Vec<_> = (0..4) + .map(|_| { + scope.spawn(|| { + barrier.wait(); + TiktokenTokenizer::from_cached_ranks(name, |file| { + loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + read_packaged_ranks(file) + }) + .unwrap() + }) + }) + .collect(); + tasks + .into_iter() + .map(|task| task.join().unwrap()) + .collect::>() + }); + assert_eq!(loads.into_inner(), usize::from(name != "gpt2")); + let actual = &encoders[0]; + let expected = TiktokenTokenizer::from_name(name).unwrap(); + assert_eq!(actual.special_tokens(), expected.special_tokens()); + let specials: Vec<_> = expected.special_tokens().into_iter().collect(); + let special_text = specials.join(" "); + assert_eq!( + actual.encode_special(&special_text, &specials).unwrap(), + expected.encode_special(&special_text, &specials).unwrap() + ); + for text in [ + "", + "café 漢字 ع 🙂", + "a\r\nb\t ", + " hello 123456789", + &special_text, + ] { + let ids = expected.encode(text); + assert_eq!(actual.encode(text), ids, "{name}: {text:?}"); + assert_eq!(actual.count_tokens(text), ids.len(), "{name}: {text:?}"); + assert_eq!( + actual.decode_bytes(&ids).unwrap(), + expected.decode_bytes(&ids).unwrap() + ); + } + let cached = TiktokenTokenizer::from_cached_ranks(name, |_| { + panic!("reloaded cached vocabulary") + }) + .unwrap(); + assert_eq!(cached.encode("cached"), expected.encode("cached")); + assert_eq!(cached.name(), name); + assert!(expected.vocabulary().is_none()); + assert_vocabulary_lookups(name, actual); + } + } + + /// The token-level lookups tiktoken's Python `Encoding` exposes, checked against the + /// encoder itself and against the known vocabulary sizes. + fn assert_vocabulary_lookups(name: &str, tokenizer: &TiktokenTokenizer) { + let max_token_value = match name { + "cl100k_base" => 100_276, + "o200k_base" => 200_018, + "o200k_harmony" => 201_087, + "p50k_base" => 50_280, + "p50k_edit" => 50_283, + "r50k_base" | "gpt2" => 50_256, + _ => unreachable!("{name}"), + }; + let vocabulary = tokenizer.vocabulary().unwrap(); + assert_eq!(vocabulary.max_token_value(), max_token_value, "{name}"); + let values = vocabulary.token_byte_values(); + assert!(values.windows(2).all(|pair| pair[0] < pair[1]), "{name}"); + for piece in values.iter().step_by(997) { + let rank = vocabulary.encode_single_token(piece).unwrap(); + assert_eq!(tokenizer.decode_bytes(&[rank]).unwrap(), *piece, "{name}"); + assert!(!vocabulary.is_special_token(rank), "{name}"); + } + for (token, rank) in vocabulary.special_tokens() { + assert_eq!(vocabulary.encode_single_token(token.as_bytes()), Some(rank)); + assert!(vocabulary.is_special_token(rank), "{name}: {token}"); + } + assert_eq!(vocabulary.encode_single_token(b"<|not-a-token|>"), None); + } + + #[test] + fn malformed_ranks_return_errors_instead_of_panicking() { + for ranks in ["", "IQ==", "IQ== x", "!!! 1", "IQ== 1"] { + assert!(build("cl100k_base", ranks).is_err()); + } + let repeated_rank = (0..=u8::MAX) + .map(|byte| format!("{} 0\n", STANDARD.encode([byte]))) + .collect::(); + assert!(build("cl100k_base", &repeated_rank).is_err()); + } +} diff --git a/litellm-rust/crates/token-counter/README.md b/litellm-rust/crates/token-counter/README.md index a6b2b50aac0..3c6381fcea7 100644 --- a/litellm-rust/crates/token-counter/README.md +++ b/litellm-rust/crates/token-counter/README.md @@ -1,6 +1,10 @@ # Token counting -`Tokenizer` is the text-counting interface. `TokenCounter` applies LiteLLM request, message, and tool accounting using any implementation of that interface +`Tokenizer` is the text-counting interface. `TextCodec` adds encoding, decoding, and a name. `TokenCounter` applies LiteLLM request, message, and tool accounting using any `Tokenizer` + +Counts follow the codec: tiktoken treats special-token spellings as ordinary text, while Hugging Face applies its added tokens, post-processing, padding, and truncation. `fast=True` preserves those semantics and requests acceleration where available. Unsupported configurations use the normal codec, including tiktoken encodings without a scanner and builds without the `fast` feature. Invalid input and process-guard errors still propagate. Runtime request counting currently uses the normal codec; the custom accelerator is retained for explicit use and testing + +`FastCounter: TextCodec` exposes an optional accelerator over a loaded codec. `None` means callers should use that codec. The Python bridge caches this selection per immutable tokenizer, shares it with request counters, and initializes it with the GIL released. Hugging Face can also choose the full encoder per input when added tokens require it The `fast` feature provides `fast::FastTokenizer` from `litellm-token-counter-fast`. `TokenCounter::from_json_fast` uses this implementation @@ -8,7 +12,9 @@ The `huggingface` feature provides `huggingface::HuggingFaceTokenizer` through t The `tiktoken` feature provides `tiktoken::TiktokenTokenizer` through `tiktoken-rs`. Select an encoding with `TokenCounter::from_tiktoken`. The supported names are `cl100k_base`, `o200k_base`, `o200k_harmony`, `p50k_base`, `p50k_edit`, `r50k_base`, and `gpt2` -All three backends are enabled by default. The Python extension builds with `fast` only, which keeps the wheel at the size it had before the split. With `default-features = false`, callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend +All three backends are enabled by default in this crate and the Python extension. With `default-features = false`, Rust callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend + +Python `tiktoken` and `tokenizers` remain runtime dependencies and the default implementations. The catalog independently selects the tokenizer and request-counting routes. Enabling Rust changes factory dispatch; existing tokenizer objects keep their backend. Native Hugging Face wrappers provide an immutable encoding and decoding API, while training and mutable configuration remain available through the Python backend Budget checks, cost calculation, and the `max_tokens` adjustment policy belong to `litellm-core-utils`. The counter does not own prices, budgets, or request limits diff --git a/litellm-rust/crates/token-counter/src/error.rs b/litellm-rust/crates/token-counter/src/error.rs index b05ce007e46..6a94b0e39b8 100644 --- a/litellm-rust/crates/token-counter/src/error.rs +++ b/litellm-rust/crates/token-counter/src/error.rs @@ -32,6 +32,8 @@ pub enum Error { JsonUtf8(#[source] FromUtf8Error), #[error("tokenization failed: {0}")] Encode(#[source] Box), + #[error("token decoding failed: {0}")] + Decode(String), #[error("token counting task failed: {0}")] Task(String), } diff --git a/litellm-rust/crates/token-counter/src/fast.rs b/litellm-rust/crates/token-counter/src/fast.rs index de3f86abd68..f601c3cbcca 100644 --- a/litellm-rust/crates/token-counter/src/fast.rs +++ b/litellm-rust/crates/token-counter/src/fast.rs @@ -1,7 +1,31 @@ use litellm_token_counter_fast::Error as BackendError; pub use litellm_token_counter_fast::FastTokenizer; -use crate::{Error, TokenCounter, Tokenizer}; +use crate::{Error, TextCodec, TokenCounter, Tokenizer}; + +pub trait FastCounter: TextCodec { + fn fast_counter(&self) -> Option; +} + +#[cfg(feature = "huggingface")] +impl FastCounter for crate::huggingface::HuggingFaceTokenizer { + fn fast_counter(&self) -> Option { + Some(FastTokenizer::from_shared(self.shared())) + } +} + +#[cfg(feature = "tiktoken")] +impl FastCounter for crate::tiktoken::TiktokenTokenizer { + fn fast_counter(&self) -> Option { + let vocabulary = self.vocabulary()?; + match self.name() { + "cl100k_base" => FastTokenizer::from_cl100k_pairs(vocabulary.ranks()), + "o200k_base" | "o200k_harmony" => FastTokenizer::from_o200k_pairs(vocabulary.ranks()), + _ => return None, + } + .ok() + } +} impl TokenCounter { pub fn from_json_fast(tokenizer_json: &str) -> Result { @@ -39,3 +63,65 @@ impl From for Error { } } } + +#[cfg(all(test, feature = "huggingface", feature = "tiktoken"))] +mod tests { + use super::*; + use crate::huggingface::HuggingFaceTokenizer; + use crate::tiktoken::TiktokenTokenizer; + + const TEXTS: [&str; 4] = [ + "", + "hello world <|endoftext|>", + "café 漢字 ع 🙂 line\r\n indented 123456789", + "system a\u{301} fi", + ]; + + fn packaged(file: &str) -> String { + std::fs::read_to_string(format!( + "{}/../../../litellm/litellm_core_utils/tokenizers/{file}", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap() + } + + #[test] + fn fast_counters_derived_from_codecs_count_like_the_codecs() { + let huggingface = + HuggingFaceTokenizer::from_json(&packaged("anthropic_tokenizer.json")).unwrap(); + let fast = huggingface.fast_counter().unwrap(); + for text in TEXTS { + assert_eq!( + fast.count_tokens(text).unwrap(), + Tokenizer::count_tokens(&huggingface, text).unwrap(), + "{text:?}" + ); + } + + for name in ["cl100k_base", "o200k_base", "o200k_harmony"] { + let tiktoken = + TiktokenTokenizer::from_cached_ranks(name, |file| Ok(packaged(file))).unwrap(); + let fast = tiktoken.fast_counter().unwrap(); + for text in TEXTS { + assert_eq!( + fast.count_tokens(text).unwrap(), + tiktoken.count_tokens(text), + "{name}: {text:?}" + ); + } + } + } + + #[test] + fn encodings_without_a_fast_scanner_keep_the_codec() { + let tiktoken = + TiktokenTokenizer::from_cached_ranks("p50k_base", |file| Ok(packaged(file))).unwrap(); + assert!(tiktoken.fast_counter().is_none()); + assert!( + TiktokenTokenizer::from_name("cl100k_base") + .unwrap() + .fast_counter() + .is_none() + ); + } +} diff --git a/litellm-rust/crates/token-counter/src/huggingface.rs b/litellm-rust/crates/token-counter/src/huggingface.rs index fb7683b373e..43613fac8ea 100644 --- a/litellm-rust/crates/token-counter/src/huggingface.rs +++ b/litellm-rust/crates/token-counter/src/huggingface.rs @@ -1,7 +1,11 @@ use litellm_token_counter_huggingface::Error as BackendError; -pub use litellm_token_counter_huggingface::HuggingFaceTokenizer; +pub use litellm_token_counter_huggingface::{ + AddedToken, EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection, + PaddingParams, PaddingStrategy, TruncationDirection, TruncationParams, encoding_from_json, + encoding_to_json, +}; -use crate::{Error, TokenCounter, Tokenizer}; +use crate::{Error, TextCodec, TokenCounter, Tokenizer}; impl TokenCounter { pub fn from_json(tokenizer_json: &str) -> Result { @@ -17,11 +21,26 @@ impl Tokenizer for HuggingFaceTokenizer { } } +impl TextCodec for HuggingFaceTokenizer { + fn encode(&self, text: &str) -> Result, Error> { + HuggingFaceTokenizer::encode(self, text).map_err(Error::from) + } + + fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result { + HuggingFaceTokenizer::decode(self, ids, skip_special_tokens).map_err(Error::from) + } + + fn name(&self) -> &str { + HuggingFaceTokenizer::name(self) + } +} + impl From for Error { fn from(error: BackendError) -> Self { match error { BackendError::Load(source) => Self::Load(source), BackendError::Encode(source) => Self::Encode(source), + BackendError::Decode(source) => Self::Decode(source.to_string()), } } } diff --git a/litellm-rust/crates/token-counter/src/lib.rs b/litellm-rust/crates/token-counter/src/lib.rs index 446c91049de..84bf35ca2ba 100644 --- a/litellm-rust/crates/token-counter/src/lib.rs +++ b/litellm-rust/crates/token-counter/src/lib.rs @@ -20,5 +20,5 @@ pub mod tiktoken; pub use counter::{InputTokenCount, TokenCounter}; pub use error::Error; -pub use tokenizer::Tokenizer; +pub use tokenizer::{TextCodec, Tokenizer}; pub use types::CountableRequest; diff --git a/litellm-rust/crates/token-counter/src/tiktoken.rs b/litellm-rust/crates/token-counter/src/tiktoken.rs index 07c1c9f5b73..5883a629ea5 100644 --- a/litellm-rust/crates/token-counter/src/tiktoken.rs +++ b/litellm-rust/crates/token-counter/src/tiktoken.rs @@ -1,7 +1,7 @@ -pub use litellm_token_counter_tiktoken::TiktokenTokenizer; -use litellm_token_counter_tiktoken::UnsupportedTokenizer; +use litellm_token_counter_tiktoken::{LoadError, UnsupportedTokenizer}; +pub use litellm_token_counter_tiktoken::{TiktokenTokenizer, Vocabulary, encoding_for_model}; -use crate::{Error, TokenCounter, Tokenizer}; +use crate::{Error, TextCodec, TokenCounter, Tokenizer}; impl TokenCounter { pub fn from_tiktoken(encoding: &str) -> Result { @@ -17,8 +17,31 @@ impl Tokenizer for TiktokenTokenizer { } } +impl TextCodec for TiktokenTokenizer { + fn encode(&self, text: &str) -> Result, Error> { + Ok(TiktokenTokenizer::encode(self, text)) + } + + fn decode(&self, ids: &[u32], _skip_special_tokens: bool) -> Result { + TiktokenTokenizer::decode(self, ids).map_err(|error| Error::Decode(error.to_string())) + } + + fn name(&self) -> &str { + TiktokenTokenizer::name(self) + } +} + impl From for Error { fn from(error: UnsupportedTokenizer) -> Self { Self::UnsupportedTokenizer(error.0) } } + +impl From for Error { + fn from(error: LoadError) -> Self { + match error { + LoadError::Unsupported(error) => error.into(), + LoadError::Ranks(message) => Self::Ranks(message), + } + } +} diff --git a/litellm-rust/crates/token-counter/src/tokenizer.rs b/litellm-rust/crates/token-counter/src/tokenizer.rs index 88c29c672a7..146ac5b4d0d 100644 --- a/litellm-rust/crates/token-counter/src/tokenizer.rs +++ b/litellm-rust/crates/token-counter/src/tokenizer.rs @@ -4,6 +4,12 @@ pub trait Tokenizer: Send + Sync { fn count_tokens(&self, text: &str) -> Result; } +pub trait TextCodec: Tokenizer { + fn encode(&self, text: &str) -> Result, Error>; + fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result; + fn name(&self) -> &str; +} + #[cfg(test)] mod tests { use super::*; diff --git a/litellm/__init__.py b/litellm/__init__.py index 44515472648..471b273f00d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -381,6 +381,7 @@ enable_model_config_credential_overrides: bool = False enable_key_alias_format_validation: bool = ( False # opt-in validation of key_alias format on /key/generate and /key/update ) +key_alias_pattern: str | None = None enable_gemini_default_thinking_level_low: bool = ( False # opt-in: force thinkingLevel low/minimal for Gemini 3 thinking param mapping ) @@ -2146,6 +2147,10 @@ if TYPE_CHECKING: from .llms.edenai.videos.transformation import ( EdenAIVideoConfig as EdenAIVideoConfig, ) + from .llms.fal_ai.chat.transformation import ( + FalAIChatConfig as FalAIChatConfig, + FalAIError as FalAIError, + ) from .llms.ovhcloud.chat.transformation import ( OVHCloudChatConfig as OVHCloudChatConfig, ) diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index fe3c7c264ee..29fb46fa125 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -58,7 +58,8 @@ from ._lazy_imports_registry import ( if TYPE_CHECKING: import httpx - from tiktoken import Encoding + + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer def get_litellm_globals() -> dict[str, object]: @@ -89,26 +90,11 @@ def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "flo # These are special lazy loaders for things that are used internally # They're separate from the main lazy import system because they have specific use cases -# Lazy loader for default encoding - avoids importing heavy tiktoken library at startup -_default_encoding: "Encoding | None" = None +def _get_default_encoding() -> "Tokenizer": + from litellm.rust_bridge.tokenizer import get_encoding -def _get_default_encoding() -> "Encoding": - """ - Lazily load and cache the default OpenAI encoding. - - This avoids importing `litellm.litellm_core_utils.default_encoding` (and thus tiktoken) - at `litellm` import time. The encoding is cached after the first import. - - This is used internally by utils.py functions that need the encoding but shouldn't - trigger its import during module load. - """ - global _default_encoding - if _default_encoding is None: - from litellm.litellm_core_utils.default_encoding import encoding - - _default_encoding = encoding - return _default_encoding + return get_encoding("cl100k_base") # Lazy loader for get_modified_max_tokens to avoid importing token_counter at module import time diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index db4eb8bdb33..9a53273c9d5 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -335,6 +335,8 @@ LLM_CONFIG_NAMES: Final = ( "EdenAITextToSpeechConfig", "EdenAIImageGenerationConfig", "EdenAIVideoConfig", + "FalAIChatConfig", + "FalAIError", "OVHCloudChatConfig", "OVHCloudEmbeddingConfig", "CometAPIEmbeddingConfig", @@ -1251,6 +1253,8 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "EdenAITextToSpeechConfig": (".llms.edenai.text_to_speech.transformation", "EdenAITextToSpeechConfig"), "EdenAIImageGenerationConfig": (".llms.edenai.image_generation.transformation", "EdenAIImageGenerationConfig"), "EdenAIVideoConfig": (".llms.edenai.videos.transformation", "EdenAIVideoConfig"), + "FalAIChatConfig": (".llms.fal_ai.chat.transformation", "FalAIChatConfig"), + "FalAIError": (".llms.fal_ai.chat.transformation", "FalAIError"), "OVHCloudChatConfig": (".llms.ovhcloud.chat.transformation", "OVHCloudChatConfig"), "OVHCloudEmbeddingConfig": ( ".llms.ovhcloud.embedding.transformation", diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 917bfbd5ae9..3a28d65e47c 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -7,6 +7,7 @@ "bash_20250124": null, "code-execution-2025-08-25": "code-execution-2025-08-25", "compact-2026-01-12": "compact-2026-01-12", + "compact-2026-09-04": "compact-2026-09-04", "computer-use-2025-01-24": "computer-use-2025-01-24", "computer-use-2025-11-24": "computer-use-2025-11-24", "context-1m-2025-08-07": "context-1m-2025-08-07", diff --git a/litellm/anthropic_interface/exceptions/exceptions.py b/litellm/anthropic_interface/exceptions/exceptions.py index 91bcf82f455..b48cd2fee6f 100644 --- a/litellm/anthropic_interface/exceptions/exceptions.py +++ b/litellm/anthropic_interface/exceptions/exceptions.py @@ -25,6 +25,7 @@ class AnthropicErrorDetail(TypedDict): type: AnthropicErrorType message: str provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]] + litellm_call_id: NotRequired[ReadOnly[str]] class AnthropicErrorResponse(TypedDict, total=False): diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 6a98b104221..a31cad4af29 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -15,7 +15,8 @@ import time import traceback from collections.abc import Mapping from enum import Enum -from typing import Any, Final +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final from pydantic import BaseModel @@ -38,6 +39,25 @@ from .redis_cluster_cache import RedisClusterCache from .redis_semantic_cache import RedisSemanticCache from .s3_cache import S3Cache +if TYPE_CHECKING: + from litellm.rust_bridge.response_cache import NativeCacheRequest, ResponseCacheRuntime + + +def _native_response(result: object) -> object: + """The value Python's own reader would return for `result` once it is cached. + + Python stores a model as its JSON text and `json.loads` a string response on read, so the + native store receives the decoded value and writes the envelope shape Python reads. + """ + if isinstance(result, BaseModel): + return json.loads(result.model_dump_json()) + if isinstance(result, str): + try: + return json.loads(result) + except ValueError: + return result + return result + def print_verbose(print_statement): try: @@ -55,6 +75,8 @@ class CacheMode(str, Enum): #### LiteLLM.Completion / Embedding Cache #### class Cache: + _native_cache: "ResponseCacheRuntime | None" = None + def __init__( self, type: LiteLLMCacheType | None = LiteLLMCacheType.LOCAL, @@ -292,6 +314,12 @@ class Cache: if self.namespace is not None and isinstance(self.cache, RedisCache): self.cache.namespace = self.namespace + from litellm.rust_bridge.response_cache import resolve_response_cache + + # The Rust catalog picks the store per backend. When it selects Rust, the storage calls + # below go to the native runtime and the Python backend stays only for its direct API. + self._native_cache = resolve_response_cache(self) + # Params whose values carry prompt content. Excluded from semantic-cache # scope keys so differently worded prompts share a bucket and match via # vector similarity rather than being split into per-wording buckets. @@ -570,6 +598,13 @@ class Cache: if "semantic-similarity" in cache_lookup_metadata: original_metadata["semantic-similarity"] = cache_lookup_metadata["semantic-similarity"] + @staticmethod + def _stamp_semantic_similarity(kwargs: Mapping[str, object], similarity: float | None) -> None: + """Write a native semantic lookup's similarity where the Python backends put it.""" + metadata: Final = kwargs.get("metadata") + if similarity is not None and isinstance(metadata, dict): + metadata["semantic-similarity"] = similarity + def get_cache(self, dynamic_cache_object: BaseCache | None = None, **kwargs): """ Retrieves the cached result for the given arguments. @@ -588,6 +623,15 @@ class Cache: cache_key = kwargs["cache_key"] else: cache_key = self.get_cache_key(**kwargs) + if cache_key is not None and self._native_cache is not None: + request = self._native_cache.request(self, MappingProxyType({**kwargs, "cache_key": cache_key})) + if request is None: + return None + if not self._is_semantic_cache(): + return self._native_cache.lookup(request) + response, similarity = self._native_cache.lookup_semantic(request) + self._stamp_semantic_similarity(kwargs, similarity) + return response if cache_key is not None: cache_control_args: Final[DynamicCacheControl] = kwargs.get("cache", {}) max_age = cache_control_args.get("s-maxage") or cache_control_args.get("s-max-age") or float("inf") @@ -620,6 +664,15 @@ class Cache: cache_key = kwargs["cache_key"] else: cache_key = self.get_cache_key(**kwargs) + if cache_key is not None and self._native_cache is not None: + request = self._native_cache.request(self, MappingProxyType({**kwargs, "cache_key": cache_key})) + if request is None: + return None + if not self._is_semantic_cache(): + return await self._native_cache.async_lookup(request) + response, similarity = await self._native_cache.async_lookup_semantic(request) + self._stamp_semantic_similarity(kwargs, similarity) + return response if cache_key is not None: cache_control_args: Final = kwargs.get("cache", {}) max_age: Final = cache_control_args.get("s-max-age", cache_control_args.get("s-maxage", float("inf"))) @@ -676,6 +729,11 @@ class Cache: try: if self.should_use_cache(**kwargs) is not True: return + if self._native_cache is not None: + request = self._native_request(kwargs) + if request is not None: + self._native_cache.store(request, _native_response(result)) + return cache_key, cached_data, kwargs = self._add_cache_logic(result=result, **kwargs) self.cache.set_cache(cache_key, cached_data, **kwargs) except Exception as e: @@ -695,6 +753,11 @@ class Cache: try: if self.should_use_cache(**kwargs) is not True: return + if self._native_cache is not None: + request = self._native_request(kwargs) + if request is not None: + await self._native_cache.async_store(request, _native_response(result)) + return if self.type == "redis" and self.redis_flush_size is not None: # high traffic - fill in results in memory and then flush await self.batch_cache_write(result, **kwargs) @@ -718,35 +781,23 @@ class Cache: Convert any embedding response into the standardized CachedEmbedding TypedDict format. """ try: - if isinstance(embedding_response, dict): - return { - "embedding": embedding_response.get("embedding"), - "index": embedding_response.get("index"), - "object": embedding_response.get("object"), - "model": model, - "prompt_tokens": prompt_tokens, - "prompt_tokens_details": prompt_tokens_details, - } - elif hasattr(embedding_response, "model_dump"): - data = embedding_response.model_dump() - return { - "embedding": data.get("embedding"), - "index": data.get("index"), - "object": data.get("object"), - "model": model, - "prompt_tokens": prompt_tokens, - "prompt_tokens_details": prompt_tokens_details, - } - else: - data = vars(embedding_response) - return { - "embedding": data.get("embedding"), - "index": data.get("index"), - "object": data.get("object"), - "model": model, - "prompt_tokens": prompt_tokens, - "prompt_tokens_details": prompt_tokens_details, - } + data: Final = ( + embedding_response + if isinstance(embedding_response, dict) + else embedding_response.model_dump() + if hasattr(embedding_response, "model_dump") + else vars(embedding_response) + ) + cached: Final[CachedEmbedding] = { + "embedding": data.get("embedding"), + "index": data.get("index"), + "object": data.get("object"), + "model": model, + "prompt_tokens": prompt_tokens, + "prompt_tokens_details": prompt_tokens_details, + "format_version": EMBEDDING_CACHE_FORMAT_VERSION, + } + return cached except KeyError as e: raise ValueError(f"Missing expected key in embedding response: {e}") @@ -862,6 +913,15 @@ class Cache: if self.should_use_cache(**kwargs) is not True: return + input_count: Final = len(kwargs["input"]) if isinstance(kwargs["input"], list) else 1 + if len(result.data) != input_count: + verbose_logger.debug( + "LiteLLM Cache: skipping embedding cache write, %d inputs but %d embeddings in the response", + input_count, + len(result.data), + ) + return + # set default ttl if not set if self.ttl is not None: kwargs["ttl"] = self.ttl @@ -879,13 +939,35 @@ class Cache: cache_key, cached_data, kwargs = self.add_embedding_response_to_cache(result, kwargs["input"], kwargs) cache_list.append((cache_key, cached_data)) - if dynamic_cache_object is not None: + if self._native_cache is not None: + entries: Final = tuple( + (request, cached_data["response"]) + for cache_key, cached_data in cache_list + if (request := self._native_request(MappingProxyType({**kwargs, "cache_key": cache_key}))) + is not None + ) + await self._native_cache.async_store_batch( + tuple(request for request, _ in entries), + tuple(response for _, response in entries), + ) + elif dynamic_cache_object is not None: await dynamic_cache_object.async_set_cache_pipeline(cache_list=cache_list, **kwargs) else: await self.cache.async_set_cache_pipeline(cache_list=cache_list, **kwargs) except Exception as e: self._log_add_cache_failure(e) + def _native_request(self, kwargs: Mapping[str, object]) -> "NativeCacheRequest | None": + if self._native_cache is None: + return None + cache_key: Final = kwargs.get("cache_key") + return self._native_cache.request( + self, + kwargs + if isinstance(cache_key, str) + else MappingProxyType({**kwargs, "cache_key": self.get_cache_key(**kwargs)}), + ) + def should_use_cache(self, **kwargs): """ Returns true if we should use the cache for LLM API calls diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 38fc8df3c85..e080951ad38 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -22,7 +22,7 @@ from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, ValidationError import litellm from litellm._logging import print_verbose, verbose_logger @@ -35,7 +35,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( from litellm.litellm_core_utils.logging_utils import ( _assemble_complete_response_from_streaming_chunks, ) -from litellm.types.caching import CachedEmbedding +from litellm.types.caching import EMBEDDING_CACHE_FORMAT_VERSION, CachedEmbedding from litellm.types.integrations.custom_logger import converted_stream_requested from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.rerank import RerankResponse @@ -78,7 +78,7 @@ class CachingHandlerResponse(BaseModel): cached_result: object | None = None final_embedding_cached_response: EmbeddingResponse | None = None embedding_all_elements_cache_hit: bool = False # this is set to True when all elements in the list have a cache hit in the embedding cache, if true return the final_embedding_cached_response no need to make an API call - remaining_embedding_inputs: tuple[str | int | list[int], ...] | None = None + embedding_uncached_input: list[str | int | list[int]] | None = None in_memory_cache_obj: Final = InMemoryCache() @@ -170,6 +170,37 @@ def _request_cache_key(request_kwargs: Mapping[str, Any]) -> str | None: return request_kwargs.get("cache_key", None) +class _CachedEmbeddingRecord(BaseModel): + model_config = ConfigDict(frozen=True) + + embedding: list[float] | str | None + index: int | None + object: str | None + model: str | None + prompt_tokens: int | None + prompt_tokens_details: dict | None + format_version: int + + +def _current_format_embedding_entry(entry: object) -> CachedEmbedding | None: + try: + record: Final = _CachedEmbeddingRecord.model_validate(entry) + except ValidationError: + return None + if record.format_version != EMBEDDING_CACHE_FORMAT_VERSION: + return None + cached: Final[CachedEmbedding] = { + "embedding": record.embedding, + "index": record.index, + "object": record.object, + "model": record.model, + "prompt_tokens": record.prompt_tokens, + "prompt_tokens_details": record.prompt_tokens_details, + "format_version": record.format_version, + } + return cached + + class LLMCachingHandler: def __init__( self, @@ -328,9 +359,7 @@ class LLMCachingHandler: return CachingHandlerResponse( final_embedding_cached_response=final_embedding_cached_response, embedding_all_elements_cache_hit=embedding_all_elements_cache_hit, - remaining_embedding_inputs=( - remaining_embedding_inputs if final_embedding_cached_response is not None else None - ), + embedding_uncached_input=list(remaining_embedding_inputs), ) verbose_logger.debug("CACHE RESULT: %s", cached_result) @@ -666,31 +695,30 @@ class LLMCachingHandler: if _caching_handler_response.final_embedding_cached_response is None: return embedding_response - api_items: Final = iter(embedding_response.data) - final_data_list: Final = list( # mutable-ok: EmbeddingResponse exposes data as a mutable list - item - if item is not None - else Embedding.model_validate(next(api_items)).model_copy(update={"index": response_index}) - for response_index, item in enumerate(_caching_handler_response.final_embedding_cached_response.data) + cached: Final = _caching_handler_response.final_embedding_cached_response + fresh_items: Final = iter(embedding_response.data or ()) + merged_usage: Final = ( + self.combine_usage(usage1=cached.usage, usage2=embedding_response.usage) + if cached.usage is not None and embedding_response.usage is not None + else cached.usage ) - - _caching_handler_response.final_embedding_cached_response.data = final_data_list - _caching_handler_response.final_embedding_cached_response._hidden_params["cache_hit"] = True - _caching_handler_response.final_embedding_cached_response._response_ms = ( - end_time - start_time - ).total_seconds() * 1000 - - ## USAGE - if ( - _caching_handler_response.final_embedding_cached_response.usage is not None - and embedding_response.usage is not None - ): - _caching_handler_response.final_embedding_cached_response.usage = self.combine_usage( - usage1=_caching_handler_response.final_embedding_cached_response.usage, - usage2=embedding_response.usage, - ) - - return _caching_handler_response.final_embedding_cached_response + merged: Final = EmbeddingResponse( + model=cached.model, + data=[ # mutable-ok: EmbeddingResponse.data is a pydantic list field + item + if item is not None + else Embedding(embedding=next(fresh_items)["embedding"], index=position, object="embedding") + for position, item in enumerate(cached.data) + ], + usage=merged_usage, + hidden_params={ # mutable-ok: EmbeddingResponse._hidden_params is a mutable dict field + **cached._hidden_params, + "cache_hit": True, + }, + _response_headers=cached._response_headers, + ) + merged._response_ms = (end_time - start_time).total_seconds() * 1000 + return merged def _async_log_cache_hit_on_callbacks( self, @@ -778,7 +806,7 @@ class LLMCachingHandler: dynamic_cache_object=self.dual_cache, ) ) - cached_result = await asyncio.gather(*tasks) + cached_result = [_current_format_embedding_entry(entry) for entry in await asyncio.gather(*tasks)] ## check if cached result is None ## if cached_result is not None and isinstance(cached_result, list): # set cached_result to None if all elements are None diff --git a/litellm/chat_completions/dispatch.py b/litellm/chat_completions/dispatch.py index d36c0343988..8e4b7f82eb8 100644 --- a/litellm/chat_completions/dispatch.py +++ b/litellm/chat_completions/dispatch.py @@ -4,7 +4,7 @@ from types import MappingProxyType from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable from litellm import main -from litellm.rust_bridge.catalog import Context, Delivery, Route +from litellm.rust_bridge.catalog import Delivery, Route, RouteContext from litellm.rust_bridge.chat_completions.entrypoints import ( NATIVE_ACOMPLETION, NATIVE_COMPLETION, @@ -72,8 +72,8 @@ def _public_request( ) -def _context(request: LiteLLMChatCompletionsRequest) -> Context: - return Context( +def _context(request: LiteLLMChatCompletionsRequest) -> RouteContext: + return RouteContext( Route.CHAT_COMPLETIONS, provider=request.custom_llm_provider, model=request.model, diff --git a/litellm/constants.py b/litellm/constants.py index 1012ca831f2..e5b662bd515 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -238,6 +238,9 @@ LOGS_GUARDRAIL_INFORMATION_MARKER: Final = "_litellm_logs_guardrail_information" # llm_provider stamped on proxy-side rate limit errors when the model resolves to no deployment PROXY_LLM_PROVIDER_FALLBACK: Final = "litellm_proxy" +# litellm_params flag on failure logs for requests the proxy rejected before routing to a deployment +PROXY_REJECTED_BEFORE_ROUTING_KEY: Final = "proxy_rejected_before_routing" + # Generic fallback for unknown models DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = int( os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index b317e356e1d..d66d80a564b 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -33,6 +33,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( _get_service_tier_cost_key, calculate_cost_component, generic_cost_per_token, + get_batch_cost_rates, get_billable_input_tokens, get_token_type_cost_breakdown, parse_prompt_tokens_details, @@ -115,6 +116,7 @@ from litellm.types.utils import ( LlmProviders, LlmProvidersSet, ModelInfo, + ModelInfoBase, PromptTokensDetailsWrapper, ServiceTier, StandardBuiltInToolsParams, @@ -322,6 +324,48 @@ class OCRPricing(TypedDict, total=False): annotation_cost_per_page: ReadOnly[float | None] +_WALL_CLOCK_PRICED_MODES: Final = frozenset({"chat", "completion", "embedding", "responses"}) + + +def _has_token_or_tiered_pricing(model_info: ModelInfoBase) -> bool: + return ( + (model_info.get("input_cost_per_token") or 0.0) > 0 + or (model_info.get("output_cost_per_token") or 0.0) > 0 + or model_info.get("tiered_pricing") is not None + ) + + +def _bills_wall_clock_seconds(model_info: ModelInfoBase) -> bool: + mode: Final = model_info.get("mode") + return mode is None or mode in _WALL_CLOCK_PRICED_MODES + + +def _per_second_pricing_cost( + model: str, + custom_llm_provider: str | None, + response_time_ms: float | None, +) -> tuple[float, float] | None: + try: + model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model + return None + if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info): + return None + input_cost_per_second: Final = model_info.get("input_cost_per_second") + output_cost_per_second: Final = model_info.get("output_cost_per_second") + if input_cost_per_second is None and output_cost_per_second is None: + return None + seconds: Final = (response_time_ms or 0.0) / 1000 + verbose_logger.debug( + "For model=%s - input_cost_per_second: %s; output_cost_per_second: %s; response time: %s", + model, + input_cost_per_second, + output_cost_per_second, + response_time_ms, + ) + return (input_cost_per_second or 0.0) * seconds, (output_cost_per_second or 0.0) * seconds + + def cost_per_token( model: str = "", prompt_tokens: int = 0, @@ -448,9 +492,6 @@ def cost_per_token( if response_cost is not None: return response_cost[0], response_cost[1] - # given - prompt_tokens_cost_usd_dollar: float = 0 - completion_tokens_cost_usd_dollar: float = 0 model_cost_ref: Final = litellm.model_cost # Only callers that explicitly pass `custom_llm_provider` get the # dedup/prefix-join treatment. When provider is omitted, preserve legacy @@ -611,6 +652,14 @@ def cost_per_token( number_of_queries=number_of_queries or 1, optional_params=(getattr(response, "_hidden_params", None) if response else None), ) + elif ( + per_second_cost := _per_second_pricing_cost( + model=model, + custom_llm_provider=custom_llm_provider, + response_time_ms=response_time_ms, + ) + ) is not None: + return per_second_cost elif custom_llm_provider == "vertex_ai": cost_router: Final = google_cost_router( model=model_without_prefix, @@ -685,12 +734,7 @@ def cost_per_token( ) else: model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) - - if ( - (model_info.get("input_cost_per_token") or 0.0) > 0 - or (model_info.get("output_cost_per_token") or 0.0) > 0 - or model_info.get("tiered_pricing") is not None - ): + if _has_token_or_tiered_pricing(model_info): return generic_cost_per_token( model=model, usage=usage_block, @@ -698,36 +742,8 @@ def cost_per_token( service_tier=service_tier, data_residency=data_residency, ) - - input_cost_per_second: Final = model_info.get("input_cost_per_second") - if input_cost_per_second is not None and response_time_ms is not None: - verbose_logger.debug( - "For model=%s - input_cost_per_second: %s; response time: %s", - model, - input_cost_per_second, - response_time_ms, - ) - ## COST PER SECOND ## - prompt_tokens_cost_usd_dollar = input_cost_per_second * response_time_ms / 1000 - - output_cost_per_second: Final = model_info.get("output_cost_per_second") - if output_cost_per_second is not None and response_time_ms is not None: - verbose_logger.debug( - "For model=%s - output_cost_per_second: %s; response time: %s", - model, - output_cost_per_second, - response_time_ms, - ) - ## COST PER SECOND ## - completion_tokens_cost_usd_dollar = output_cost_per_second * response_time_ms / 1000 - - verbose_logger.debug( - "Returned custom cost for model=%s - prompt_tokens_cost_usd_dollar: %s, completion_tokens_cost_usd_dollar: %s", - model, - prompt_tokens_cost_usd_dollar, - completion_tokens_cost_usd_dollar, - ) - return prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar + verbose_logger.debug("No per-token, tiered, or per-second pricing for model=%s; cost is 0", model) + return 0.0, 0.0 def get_replicate_completion_pricing(completion_response: dict, total_time=0.0): @@ -948,7 +964,7 @@ def _extract_service_tier(source: object) -> str | None: return None -def _get_usage_object( +def get_usage_object( completion_response: object, ) -> Usage | None: usage_obj: Final = cast( @@ -1222,6 +1238,21 @@ def _split_responses_ws_logging_object_by_service_tier( ) +def _response_time_ms_for_cost( + completion_response: object, + litellm_logging_obj: LitellmLoggingObject | None, + total_time: float | None, +) -> float: + stamped: Final = getattr(completion_response, "_response_ms", None) + if isinstance(stamped, (int, float)): + return float(stamped) + if total_time: + return total_time + if litellm_logging_obj is not None: + return litellm_logging_obj.get_response_ms() + return 0.0 + + def completion_cost( completion_response: object | None = None, model: str | None = None, @@ -1336,7 +1367,7 @@ def completion_cost( cache_creation_input_tokens: int | None = None cache_read_input_tokens: int | None = None audio_transcription_file_duration: float = 0.0 - provider_usage_object: Final = _get_usage_object(completion_response=completion_response) + provider_usage_object: Final = get_usage_object(completion_response=completion_response) cost_per_token_usage_object: Final[Usage | None] = ( _without_provider_stated_cost(provider_usage_object) if custom_pricing else provider_usage_object ) @@ -1443,8 +1474,6 @@ def completion_cost( prompt_tokens_details = _usage.get("prompt_tokens_details") or {} cache_read_input_tokens = prompt_tokens_details.get("cached_tokens", 0) - total_time = getattr(completion_response, "_response_ms", 0) - hidden_params = getattr(completion_response, "_hidden_params", None) if hidden_params is not None: custom_llm_provider = hidden_params.get("custom_llm_provider", custom_llm_provider or None) @@ -1503,6 +1532,7 @@ def completion_cost( size=size, optional_params=optional_params, call_type=call_type, + model_info=_deployment_model_info(litellm_logging_obj, custom_pricing, router_model_id), ) elif call_type in _VIDEO_CALL_TYPES: ### VIDEO GENERATION COST CALCULATION ### @@ -1676,6 +1706,11 @@ def completion_cost( ) return MCPCostCalculator.calculate_mcp_tool_call_cost(litellm_logging_obj=litellm_logging_obj) + response_time_ms = _response_time_ms_for_cost( + completion_response=completion_response, + litellm_logging_obj=litellm_logging_obj, + total_time=total_time, + ) # Calculate cost based on prompt_tokens, completion_tokens if ( "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai" @@ -1686,7 +1721,7 @@ def completion_cost( # see https://replicate.com/pricing elif (model in litellm.replicate_models or "replicate" in model) and model not in litellm.model_cost: # for unmapped replicate model, default to replicate's time tracking logic - return get_replicate_completion_pricing(completion_response, total_time) + return get_replicate_completion_pricing(completion_response, response_time_ms) if model is None: raise ValueError( @@ -1718,7 +1753,7 @@ def completion_cost( prompt_tokens=prompt_tokens or 0, completion_tokens=completion_tokens or 0, custom_llm_provider=custom_llm_provider, - response_time_ms=total_time, + response_time_ms=response_time_ms, region_name=None if explicit_pricing else region_name, custom_cost_per_second=custom_cost_per_second, custom_cost_per_token=custom_cost_per_token, @@ -1977,13 +2012,9 @@ def _deployment_model_info( ) -> ModelInfo | None: if not custom_pricing: return None - registered_deployment_info: Final = ( - _cost_map_model_info(router_model_id, None) - if router_model_id is not None and router_model_id in litellm.model_cost - else None - ) + registered_deployment_info: Final = _raw_cost_map_entry(router_model_id) if router_model_id is not None else None if registered_deployment_info is not None: - return registered_deployment_info + return cast(ModelInfo, registered_deployment_info) # cast-ok: router registers deployment prices under its id if litellm_logging_obj is None: return None litellm_params: Final = getattr(litellm_logging_obj, "litellm_params", None) @@ -2033,6 +2064,44 @@ def _cost_map_model_info(model: str, custom_llm_provider: str | None) -> ModelIn return None +def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None: + raw_entry: Final = litellm.model_cost.get(key) + return raw_entry if isinstance(raw_entry, Mapping) else None + + +def pricing_entry_for_cost_calc( + model: str | None, + completion_response: object | None, + custom_llm_provider: str | None, + custom_pricing: bool | None, + base_model: str | None, + router_model_id: str | None, + region_name: str | None, + litellm_logging_obj: LitellmLoggingObject | None, +) -> tuple[str, Mapping[str, object]] | None: + deployment_entry: Final = _deployment_model_info(litellm_logging_obj, custom_pricing, router_model_id) + deployment_key: Final = router_model_id or model + if deployment_entry is not None and deployment_key is not None: + return deployment_key, deployment_entry + selected_model: Final = _select_model_name_for_cost_calc( + model=model, + completion_response=completion_response, + base_model=base_model, + custom_pricing=custom_pricing, + custom_llm_provider=custom_llm_provider, + router_model_id=router_model_id, + region_name=region_name, + ) + candidates: Final = (selected_model, _get_response_model(completion_response), model) + resolved: Final = next( + (info for info in (_cost_map_model_info(name, custom_llm_provider) for name in candidates if name) if info), + None, + ) + if resolved is None: + return None + return resolved["key"], _raw_cost_map_entry(resolved["key"]) or resolved + + def ocr_cost( model: str, custom_llm_provider: str | None, @@ -2273,6 +2342,7 @@ def default_image_cost_calculator( n: int | None = 1, # Default to 1 image size: str | None = "1024-x-1024", # OpenAI default optional_params: dict | None = None, + model_info: ModelInfo | None = None, ) -> float: """ Default image cost calculator for image generation @@ -2283,6 +2353,7 @@ def default_image_cost_calculator( quality (Optional[str]): Image quality setting n (Optional[int]): Number of images generated size (Optional[str]): Image size (e.g. "1024x1024" or "1024-x-1024") + model_info (Optional[ModelInfo]): The deployment's own prices, consulted before the cost map Returns: float: Cost in USD for the image generation @@ -2313,9 +2384,7 @@ def default_image_cost_calculator( model_without_provider: Final = f"{size_str}/{model.split('/')[-1]}" model_with_quality_without_provider = f"{quality}/{model_without_provider}" if quality else model_without_provider - # Try model with quality first, fall back to base model name - cost_info: dict | None = None - models_to_check: Final[list[str | None]] = [ + models_to_check: Final = ( model_name_with_quality, base_model_name, model_name_with_v2_quality, @@ -2323,22 +2392,33 @@ def default_image_cost_calculator( model_without_provider, model, model_name_without_custom_llm_provider, - ] - for _model in models_to_check: - if _model is not None and _model in litellm.model_cost: - cost_info = litellm.model_cost[_model] - break - if cost_info is None: + ) + matched_model: Final = next( + (_model for _model in models_to_check if _model is not None and _model in litellm.model_cost), None + ) + if matched_model is None and model_info is None: raise Exception(f"Model not found in cost map. Tried checking {models_to_check}") - # Priority 1: Use per-image pricing if available (for gpt-image-1 and similar models) - if "input_cost_per_image" in cost_info and cost_info["input_cost_per_image"] is not None: - return cost_info["input_cost_per_image"] * n - # Priority 2: Fall back to per-pixel pricing for backward compatibility - elif "input_cost_per_pixel" in cost_info and cost_info["input_cost_per_pixel"] is not None: - return cost_info["input_cost_per_pixel"] * height * width * n - else: + shared_cost_info: Final = litellm.model_cost[matched_model] if matched_model is not None else None + price_tables: Final = tuple(table for table in (model_info, shared_cost_info) if table is not None) + image_count: Final = n if n is not None else 1 + unit_counts: Final = ( + ("input_cost_per_image", image_count), + ("output_cost_per_image", image_count), + ("input_cost_per_pixel", height * width * image_count), + ) + cost: Final = next( + ( + price * units + for price_table in price_tables + for cost_key, units in unit_counts + if (price := price_table.get(cost_key)) is not None + ), + None, + ) + if cost is None: raise Exception(f"No pricing information found for model {model}. Tried checking {models_to_check}") + return cost def default_video_cost_calculator( @@ -2485,35 +2565,14 @@ def batch_cost_calculator( if not model_info: return 0.0, 0.0 - input_cost_per_token_batches: Final = model_info.get("input_cost_per_token_batches") + batch_rates: Final = get_batch_cost_rates(model_info, usage, custom_llm_provider) input_cost_per_token: Final = model_info.get("input_cost_per_token") - output_cost_per_token_batches: Final = model_info.get("output_cost_per_token_batches") output_cost_per_token: Final = model_info.get("output_cost_per_token") total_prompt_cost = 0.0 total_completion_cost = 0.0 - if input_cost_per_token_batches is not None: - batch_details: Final = parse_prompt_tokens_details(usage) - audio_tokens, image_tokens, video_tokens = ( - batch_details["audio_tokens"], - batch_details["image_tokens"], - batch_details["video_tokens"], - ) - modality_rates: Final = ( - _batch_rate(model_info, "input_cost_per_audio_token_batches", input_cost_per_token_batches), - _batch_rate(model_info, "input_cost_per_image_token_batches", input_cost_per_token_batches), - _batch_rate(model_info, "input_cost_per_video_token_batches", input_cost_per_token_batches), - ) - total_prompt_cost = sum( - tokens * rate - for tokens, rate in zip( - ( - max((usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens, 0), - audio_tokens, - image_tokens, - video_tokens, - ), - (input_cost_per_token_batches, *modality_rates), - ) + if batch_rates.input is not None: + total_prompt_cost = _batch_prompt_cost( + usage, model_info, batch_rates.input, batch_rates.cache_read, batch_rates.cache_creation ) elif input_cost_per_token: details: Final = parse_prompt_tokens_details(usage) @@ -2533,8 +2592,8 @@ def batch_cost_calculator( cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") or input_cost_per_token total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2 - if output_cost_per_token_batches is not None: - total_completion_cost = usage.completion_tokens * output_cost_per_token_batches + if batch_rates.output is not None: + total_completion_cost = usage.completion_tokens * batch_rates.output elif output_cost_per_token: total_completion_cost = ( usage.completion_tokens * (output_cost_per_token) / 2 @@ -2548,6 +2607,34 @@ def batch_cost_calculator( return total_prompt_cost, total_completion_cost +def _batch_prompt_cost( + usage: Usage, + model_info: ModelInfo, + input_rate: float, + cache_read_rate: float | None, + cache_creation_rate: float | None, +) -> float: + details: Final = parse_prompt_tokens_details(usage) + cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0 + written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0 + audio_tokens, image_tokens, video_tokens = ( + details["audio_tokens"], + details["image_tokens"], + details["video_tokens"], + ) + text_tokens: Final = max( + (usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens - cached_tokens - written_tokens, 0 + ) + return ( + text_tokens * input_rate + + audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate) + + image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate) + + video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate) + + cached_tokens * (cache_read_rate or 0.0) + + written_tokens * (cache_creation_rate or 0.0) + ) + + def _attribute_value(obj: object, name: str) -> object: return getattr(obj, name) diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 49434befd4e..1ccae8de35f 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -7,11 +7,11 @@ import base64 import hashlib import json import os -from collections.abc import Awaitable, Callable, Generator, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Generator, Sequence from contextlib import AbstractAsyncContextManager from functools import partial from types import MappingProxyType -from typing import Any, Final, TypeAlias, TypeVar +from typing import Final, TypeAlias, TypeVar, cast import anyio import httpx2 @@ -121,14 +121,14 @@ def _strip_header_whitespace(headers: dict[str, str]) -> dict[str, str]: } -def _first_non_cancelled_cause(exc: BaseException) -> BaseException | None: +def _first_non_cancelled_cause(exc: BaseException, cleanup_errors: tuple[Exception, ...] = ()) -> BaseException | None: queue: Final[list[BaseException]] = [exc] while queue: current = queue.pop(0) nested = getattr(current, "exceptions", None) if nested: queue.extend(nested) - elif not isinstance(current, asyncio.CancelledError): + elif not isinstance(current, asyncio.CancelledError) and not any(current is error for error in cleanup_errors): return current return None @@ -159,7 +159,59 @@ _ListPage = TypeVar("_ListPage", bound=PaginatedResult) _ListItem = TypeVar("_ListItem") +async def _run_bounded_cleanup(operation: Callable[[], Awaitable[TSessionResult]], deadline: float) -> TSessionResult: + async def run() -> TSessionResult: + with anyio.fail_after(max(0, deadline - anyio.current_time()), shield=True): + return await operation() + + # A cancelled asyncio.gather repeatedly forwards Task.cancel, bypassing AnyIO shields. + # Isolate only cleanup, and drain it before propagating the caller's cancellation. + task: Final = asyncio.create_task(run()) + interrupted: asyncio.CancelledError | None = None + with anyio.CancelScope(shield=True): + while not task.done(): + try: + await asyncio.shield(task) + except asyncio.CancelledError as exc: + interrupted = exc + except Exception: + break + if interrupted is not None: + if not task.cancelled(): + task.exception() + raise interrupted + return task.result() + + +class _MCPResponseStream(httpx2.AsyncByteStream): + def __init__(self, stream: httpx2.AsyncByteStream, record_error: Callable[[Exception], None]) -> None: + self._stream: Final = stream + self._record_error: Final = record_error + + async def __aiter__(self) -> AsyncIterator[bytes]: + try: + async for chunk in self._stream: + yield chunk + except Exception as error: + self._record_error(error) + raise + + async def aclose(self) -> None: + try: + await self._stream.aclose() + except Exception as error: + self._record_error(error) + raise + + class _MCPHTTPClient(httpx2.AsyncClient): + cleanup_scope: anyio.CancelScope | None = None + cleanup_errors: tuple[Exception, ...] = () + + def _record_cleanup_error(self, error: Exception) -> None: + if self.cleanup_scope is not None and self.cleanup_scope.shield: + self.cleanup_errors += (error,) + async def send( self, request: httpx2.Request, @@ -168,11 +220,29 @@ class _MCPHTTPClient(httpx2.AsyncClient): auth: AuthTypes | UseClientDefault | None = httpx2.USE_CLIENT_DEFAULT, follow_redirects: bool | UseClientDefault = httpx2.USE_CLIENT_DEFAULT, ) -> httpx2.Response: - response: Final = await super().send(request, stream=stream, auth=auth, follow_redirects=follow_redirects) - if request.method == "POST" and response.is_error and response.status_code != 404: - await response.aclose() - response.raise_for_status() - return response + if request.method == "DELETE" and self.cleanup_scope is not None: + + async def terminate() -> httpx2.Response: + termination: Final = await super(_MCPHTTPClient, self).send( + request, stream=stream, auth=auth, follow_redirects=follow_redirects + ) + await termination.aread() + return termination + + return await _run_bounded_cleanup(terminate, self.cleanup_scope.deadline) + try: + response: Final = await super().send(request, stream=stream, auth=auth, follow_redirects=follow_redirects) + if request.method == "POST" and response.is_error and response.status_code != 404: + await response.aclose() + response.raise_for_status() + if stream: + response.stream = _MCPResponseStream( + cast(httpx2.AsyncByteStream, response.stream), self._record_cleanup_error + ) + return response + except Exception as error: + self._record_cleanup_error(error) + raise class MCPSigV4Auth(httpx2.Auth): @@ -458,6 +528,7 @@ class MCPClient: self, transport_ctx: _TransportContext, operation: Callable[[ClientSession], Awaitable[TSessionResult]], + http_client: httpx2.AsyncClient | None = None, ) -> TSessionResult: """ Execute an operation within a transport and session context. @@ -466,69 +537,97 @@ class MCPClient: so that upstream MCP servers can request LLM inference (sampling), user input (elicitation), or send log messages. """ - transport: Final = await transport_ctx.__aenter__() in_flight_error: BaseException | None = None - try: - read_stream: Final = transport[0] - write_stream: Final = transport[1] - stream_error: Final[asyncio.Future[Exception]] = asyncio.get_running_loop().create_future() - - async def receive_message( - message: ServerNotification | Exception, - ) -> None: - if not isinstance(message, (ValueError, httpx2.HTTPError, OSError)): - return - if not stream_error.done(): - stream_error.set_result(message) - # The SDK closes pending requests when its message handler raises. - raise RuntimeError("MCP response stream failed") - - # Build session kwargs with optional callbacks - session_kwargs: Final[dict[str, Any]] = {} - if self._sampling_callback is not None: - session_kwargs["sampling_callback"] = self._sampling_callback - if self._elicitation_callback is not None: - session_kwargs["elicitation_callback"] = self._elicitation_callback - if self._logging_callback is not None: - session_kwargs["logging_callback"] = self._logging_callback - # The SDK drops a response stream that ends without a JSON-RPC reply, so nothing else - # ever fails the request. - session_ctx: Final = ClientSession( - read_stream, - write_stream, - read_timeout_seconds=self.timeout, - message_handler=receive_message, - **session_kwargs, - ) - session: Final = await session_ctx.__aenter__() + with anyio.CancelScope() as cleanup_scope: + if isinstance(http_client, _MCPHTTPClient): + http_client.cleanup_scope = cleanup_scope try: - init_result: Final = await session.initialize() - self._last_initialize_instructions = None - if init_result is not None: - ins: Final = getattr(init_result, "instructions", None) - if isinstance(ins, str) and ins.strip(): - self._last_initialize_instructions = ins.strip() - return await operation(session) - except MCPError: - if stream_error.done(): - raise stream_error.result() - raise - finally: + transport: Final = await transport_ctx.__aenter__() try: - await session_ctx.__aexit__(None, None, None) + read_stream: Final = transport[0] + write_stream: Final = transport[1] + stream_error: Final[asyncio.Future[Exception]] = asyncio.get_running_loop().create_future() + + async def receive_message( + message: ServerNotification | Exception, + ) -> None: + if not isinstance(message, (ValueError, httpx2.HTTPError, OSError)): + return + if not stream_error.done(): + stream_error.set_result(message) + # The SDK closes pending requests when its message handler raises. + raise RuntimeError("MCP response stream failed") + + session_kwargs: Final = { + name: callback + for name, callback in ( + ("sampling_callback", self._sampling_callback), + ("elicitation_callback", self._elicitation_callback), + ("logging_callback", self._logging_callback), + ) + if callback is not None + } + # The SDK drops a response stream that ends without a JSON-RPC reply, so nothing else + # ever fails the request. + session_ctx: Final = ClientSession( + read_stream, + write_stream, + read_timeout_seconds=self.timeout, + message_handler=receive_message, + **session_kwargs, + ) + session: Final = await session_ctx.__aenter__() + try: + init_result: Final = await session.initialize() + instructions: Final = getattr(init_result, "instructions", None) + self._last_initialize_instructions = ( + instructions.strip() or None if isinstance(instructions, str) else None + ) + result: Final = await operation(session) + except BaseException as operation_error: + in_flight_error = operation_error + if isinstance(operation_error, MCPError) and stream_error.done(): + raise stream_error.result() + raise + finally: + cleanup_scope.shield = True + cleanup_scope.deadline = anyio.current_time() + 5 + try: + await session_ctx.__aexit__(None, None, None) + except (Exception, asyncio.CancelledError) as e: + verbose_logger.debug("Error during session context exit: %s", e) + if in_flight_error is None and isinstance(e, asyncio.CancelledError): + raise except BaseException as e: - verbose_logger.debug("Error during session context exit: %s", e) - except BaseException as e: - in_flight_error = e - raise - finally: - try: - await transport_ctx.__aexit__(None, None, None) - except BaseException as exit_error: - verbose_logger.debug("Error during transport context exit: %s", exit_error) - root_cause: Final = _first_non_cancelled_cause(exit_error) - if root_cause is not None and isinstance(in_flight_error, asyncio.CancelledError): - raise root_cause from in_flight_error + in_flight_error = e + raise + finally: + cleanup_scope.shield = True + cleanup_scope.deadline = min(cleanup_scope.deadline, anyio.current_time() + 5) + try: + await transport_ctx.__aexit__(None, None, None) + except (Exception, asyncio.CancelledError) as exit_error: + verbose_logger.debug("Error during transport context exit: %s", exit_error) + if in_flight_error is None and isinstance(exit_error, asyncio.CancelledError): + raise + root_cause: Final = _first_non_cancelled_cause( + exit_error, http_client.cleanup_errors if isinstance(http_client, _MCPHTTPClient) else () + ) + if root_cause is not None and isinstance(in_flight_error, asyncio.CancelledError): + raise root_cause from in_flight_error + finally: + cleanup_scope.shield = False + if isinstance(http_client, _MCPHTTPClient): + http_client.cleanup_errors = () + http_client.cleanup_scope = None + await anyio.lowlevel.checkpoint_if_cancelled() + if cleanup_scope.cancel_called: + raise ( + in_flight_error + if in_flight_error is not None + else asyncio.CancelledError("MCP session cleanup timed out") + ) + return result async def run_with_session( self, @@ -542,10 +641,11 @@ class MCPClient: (call_tool / list_tools under raise_on_error), so an expected pass-through re-auth does not emit a warning per call; every other caller keeps the operator-visible warning.""" http_client: httpx2.AsyncClient | None = None + close_cancellation: asyncio.CancelledError | None = None try: self._last_initialize_instructions = None transport_ctx, http_client = self._create_transport_context() - return await self._execute_session_operation(transport_ctx, operation) + result: Final = await self._execute_session_operation(transport_ctx, operation, http_client=http_client) except Exception as e: read_timeout: Final = as_mcp_read_timeout(e) if read_timeout is not None: @@ -561,9 +661,16 @@ class MCPClient: finally: if http_client is not None: try: - await http_client.aclose() - except BaseException as e: + await _run_bounded_cleanup(http_client.aclose, anyio.current_time() + 1) + except (Exception, asyncio.CancelledError) as e: verbose_logger.debug("Error during http_client cleanup: %s", e) + if isinstance(e, asyncio.CancelledError): + close_cancellation = e + + if close_cancellation is not None: + raise close_cancellation + await anyio.lowlevel.checkpoint_if_cancelled() + return result def update_auth_value(self, mcp_auth_value: str | dict[str, str]) -> None: """ diff --git a/litellm/integrations/arize/arize.py b/litellm/integrations/arize/arize.py index 2e5b17185f2..4ab8d9796b2 100644 --- a/litellm/integrations/arize/arize.py +++ b/litellm/integrations/arize/arize.py @@ -4,13 +4,17 @@ arize AI is OTEL compatible this file has Arize ai specific helper functions """ +import math import os +import random +from collections.abc import Callable, Mapping from datetime import datetime from typing import TYPE_CHECKING, Any, Final +from litellm._logging import verbose_logger from litellm.integrations.arize import _utils from litellm.integrations.arize._utils import ArizeOTELAttributes -from litellm.integrations.opentelemetry import OpenTelemetry +from litellm.integrations.opentelemetry import _MAX_DYNAMIC_TRACER_PROVIDERS, OpenTelemetry, OpenTelemetryConfig from litellm.types.integrations.arize import ArizeConfig from litellm.types.services import ServiceLoggerPayload from litellm.types.utils import StandardCallbackDynamicParams @@ -26,6 +30,9 @@ else: Protocol = Any Span = Any +_SUCCESS_SAMPLING_RATE_VAR: Final = "arize_success_sampling_rate" +_ERROR_SAMPLING_RATE_VAR: Final = "arize_error_sampling_rate" + class ArizeLogger(OpenTelemetry): """ @@ -36,6 +43,26 @@ class ArizeLogger(OpenTelemetry): fighting over the global ``opentelemetry.trace`` TracerProvider singleton. """ + def __init__( + self, + config: OpenTelemetryConfig | None = None, + callback_name: str | None = None, + tracer_provider: object | None = None, + logger_provider: object | None = None, + meter_provider: object | None = None, + max_dynamic_tracer_providers: int = _MAX_DYNAMIC_TRACER_PROVIDERS, + random_draw: Callable[[], float] | None = None, + ) -> None: + super().__init__( + config=config, + callback_name=callback_name, + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + max_dynamic_tracer_providers=max_dynamic_tracer_providers, + ) + self._random_draw: Final[Callable[[], float]] = random_draw if random_draw is not None else random.random + def _init_tracing(self, tracer_provider): """ Override to always create a *private* TracerProvider for Arize. @@ -55,6 +82,72 @@ class ArizeLogger(OpenTelemetry): self.tracer = provider.get_tracer("litellm") self.span_kind = SpanKind + def _handle_success( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime, + end_time: datetime, + ) -> None: + if not self._should_export(kwargs, _SUCCESS_SAMPLING_RATE_VAR): + return + super()._handle_success(kwargs, response_obj, start_time, end_time) + + def _handle_failure( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime, + end_time: datetime, + ) -> None: + if not self._should_export(kwargs, _ERROR_SAMPLING_RATE_VAR): + return + super()._handle_failure(kwargs, response_obj, start_time, end_time) + + def _sampling_rate_for_request(self, kwargs: Mapping[str, object], var: str) -> float | None: + dynamic_params: Final = kwargs.get("standard_callback_dynamic_params") + if not isinstance(dynamic_params, Mapping): + return None + value: Final = dynamic_params.get(var) + if value is None or value in ("", "None"): + return None + try: + rate = float(value) + except (TypeError, ValueError): + verbose_logger.warning( + "ArizeLogger: %s value %r is not a number; exporting the request", + var, + value, + ) + return None + if not math.isfinite(rate) or not 0.0 <= rate <= 1.0: + verbose_logger.warning( + "ArizeLogger: %s value %r is outside 0.0..1.0; exporting the request", + var, + value, + ) + return None + return rate + + def _should_export(self, kwargs: dict[str, object], var: str) -> bool: + rate: Final = self._sampling_rate_for_request(kwargs, var) + if rate is None: + return True + otel_internal: Final = self._otel_internal_state(kwargs) + key: Final = f"arize_sampled:{var}" + cached: Final = otel_internal.get(key) + if isinstance(cached, bool): + return cached + sampled: Final = rate > 0.0 and self._random_draw() <= rate + otel_internal[key] = sampled + if not sampled: + verbose_logger.debug( + "ArizeLogger: dropping request, %s rate %r rejected the draw", + var, + rate, + ) + return sampled + def _init_otel_logger_on_litellm_proxy(self): """ Override: Arize should NOT overwrite the proxy's diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 5b5261fab6b..326abd5c6a3 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -574,11 +574,10 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac Useful if you want to modify the standard logging payload after the MCP tool call is made. - To change what the caller sends back to the MCP client, mutate ``response_obj`` - in place: every call site discards the returned object, because the - dispatcher unwraps it to ``mcp_tool_call_response`` (a raw content list, not - a ``CallToolResult``) which the tool-call paths cannot forward. Guardrails - that mask or reject tool output should use ``post_mcp_call`` instead. + Modify ``mcp_tool_call_response`` in place or return a replacement response + object to change what the caller sends back to the MCP client. Content rewrites + discard stale structured output and mark those results as tool errors. + Use ``post_mcp_call`` guardrails for schema-preserving structured redaction. """ return None diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 352fcdf90f3..991b0411ee8 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -1,6 +1,7 @@ #### What this does #### # On success, logs events to Langsmith import asyncio +import json import os import random import traceback @@ -415,7 +416,7 @@ class LangsmithLogger(CustomBatchLogger): langsmith_api_key: Final = credentials["LANGSMITH_API_KEY"] langsmith_tenant_id: Final = credentials.get("LANGSMITH_TENANT_ID") url: Final = self._add_endpoint_to_url(langsmith_api_base, "runs/batch") - headers: Final = {"x-api-key": langsmith_api_key} + headers: Final = {"x-api-key": langsmith_api_key, "Content-Type": "application/json"} if langsmith_tenant_id: headers["x-tenant-id"] = langsmith_tenant_id elements_to_log: Final = [queue_object["data"] for queue_object in queue_objects] @@ -426,7 +427,7 @@ class LangsmithLogger(CustomBatchLogger): verbose_logger.debug("[LANGSMITH MOCK] Mock mode enabled - API calls will be intercepted") response: Final = await self.async_httpx_client.post( url=url, - json={"post": elements_to_log}, + content=json.dumps({"post": elements_to_log}, default=str, allow_nan=False), headers=headers, ) response.raise_for_status() diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 180929bcfd4..749f0ce4fcb 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -26,6 +26,7 @@ from litellm.integrations.otel.model.baggage import promoted_metadata from litellm.integrations.otel.model.db_endpoint import db_span_attributes from litellm.integrations.otel.model.metadata import flatten_metadata from litellm.integrations.otel.model.semconv import Metric +from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.secret_redaction import redact_string @@ -99,6 +100,7 @@ _MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256 # Dedicated so a slow exporter shutdown cannot starve the shared logging executor. _PROVIDER_SHUTDOWN_EXECUTOR: Final = ThreadPoolExecutor(max_workers=4, thread_name_prefix="OtelProviderShutdown") + LITELLM_TRACER_NAME: Final = os.getenv("OTEL_TRACER_NAME", "litellm") LITELLM_METER_NAME: Final = os.getenv("LITELLM_METER_NAME", "litellm") LITELLM_LOGGER_NAME: Final = os.getenv("LITELLM_LOGGER_NAME", "litellm") @@ -1240,6 +1242,25 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): # End of Team/Key Based Logging Control Flow ######################################################### + def _otel_internal_state(self, kwargs: dict[str, object]) -> dict[str, object]: + """Return the request-local ``_otel_internal`` marker dict, creating it if absent.""" + litellm_params = kwargs.get("litellm_params") + if not isinstance(litellm_params, dict): + litellm_params = {} + kwargs["litellm_params"] = litellm_params + + _metadata = litellm_params.get("metadata") + if not isinstance(_metadata, dict): + _metadata = {} + litellm_params["metadata"] = _metadata + + _otel_internal = _metadata.get("_otel_internal") + if not isinstance(_otel_internal, dict): + _otel_internal = {} + _metadata["_otel_internal"] = _otel_internal + + return _otel_internal + def _emit_once(self, kwargs: dict, *scope: object) -> bool: """Return True the first time this handler is asked to emit a span for the given (handler, scope) on this kwargs; False on repeats. @@ -1264,20 +1285,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): request-local (kwargs is shared across the sync/async callbacks and lifecycle hooks for one request). """ - litellm_params = kwargs.get("litellm_params") - if not isinstance(litellm_params, dict): - litellm_params = {} - kwargs["litellm_params"] = litellm_params - - _metadata = litellm_params.get("metadata") - if not isinstance(_metadata, dict): - _metadata = {} - litellm_params["metadata"] = _metadata - - _otel_internal = _metadata.get("_otel_internal") - if not isinstance(_otel_internal, dict): - _otel_internal = {} - _metadata["_otel_internal"] = _otel_internal + _otel_internal = self._otel_internal_state(kwargs) spans_logged = _otel_internal.get("spans_logged") if not isinstance(spans_logged, dict): @@ -3084,8 +3092,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): otel_exporter, ) normalized_endpoint = self._normalize_otel_endpoint(otel_endpoint, "traces") + tls: Final = resolve_otlp_http_tls("TRACES") return BatchSpanProcessor( - OTLPSpanExporterHTTP(endpoint=normalized_endpoint, headers=_split_otel_headers), + OTLPSpanExporterHTTP( + endpoint=normalized_endpoint, + headers=_split_otel_headers, + certificate_file=tls.certificate_file, + session=tls.session, + ), ) elif otel_exporter == "otlp_grpc" or otel_exporter == "grpc": try: @@ -3166,7 +3180,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): self.OTEL_EXPORTER, normalized_endpoint, ) - return OTLPLogExporter(endpoint=normalized_endpoint, headers=_split_otel_headers) + tls: Final = resolve_otlp_http_tls("LOGS") + return OTLPLogExporter( + endpoint=normalized_endpoint, + headers=_split_otel_headers, + certificate_file=tls.certificate_file, + session=tls.session, + ) elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc": try: from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( @@ -3229,9 +3249,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): OTLPMetricExporter, ) + tls: Final = resolve_otlp_http_tls("METRICS") exporter = OTLPMetricExporter( endpoint=normalized_endpoint, headers=_split_otel_headers, + certificate_file=tls.certificate_file, + session=tls.session, ) return PeriodicExportingMetricReader(exporter, export_interval_millis=5000) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 6b673967427..20cdf9662e1 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -240,7 +240,7 @@ class OpenTelemetryV2(CustomLogger): provider: Final = resolve_logger_provider(self.config, logger_provider) if provider is None: return None - return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME)) + return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME), provider.resource) # ====================================================================== # # Proxy global registration diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index d3ad7234d93..2f337c59148 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -427,7 +427,7 @@ class LLMCallSpanData: # plain ``.get`` — no repeated ``isinstance`` guards. raw_response: Final = payload.get("response") response: Final = cast(Mapping[str, object], raw_response if isinstance(raw_response, dict) else {}) - choices_out: Final = _dicts(response.get("choices")) or _responses_choices(response) or _ocr_choices(response) + choices_out: Final = _output_choices(response) # ``finish_reasons`` is metadata, not content, so derive it from # ``choices_out`` before gating. The raw message/choice bodies are only # retained when content capture is enabled (see ``capture_span_content``); @@ -752,20 +752,133 @@ def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: return (choice,) -def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: - markdowns: Final = tuple( - text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None +def _output_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + """The response output as chat-shaped choices; images and binary bodies become size summaries, never bytes.""" + return ( + _completion_choices(response) + or _responses_choices(response) + or _ocr_choices(response) + or _transcription_choices(response) + or _moderation_choices(response) + or _rerank_choices(response) + or _search_choices(response) + or _image_choices(response) + or _binary_choices(response) ) - if not markdowns: + + +def _text_choice(content: str, finish_reason: str | None = None) -> _Choice: + message: Final[_AssistantMessage] = {"role": "assistant", "content": content, "refusal": None, "tool_calls": None} + return {"message": message, "finish_reason": finish_reason} + + +def _joined_choice(parts: tuple[str, ...]) -> tuple[_Choice, ...]: + return (_text_choice("\n\n".join(parts)),) if parts else () + + +def _text_completion_choice(choice: Mapping[str, object], text: str) -> Mapping[str, object]: + synthesized: Final = _text_choice(text, as_str(choice.get("finish_reason"))) + merged: Final = (*choice.items(), *synthesized.items()) + return {k: v for k, v in merged if k != "text"} # mutable-ok: mappers json.dumps and isinstance(dict) it + + +def _completion_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + return tuple( + _text_completion_choice(choice, text) + if "message" not in choice and isinstance(text := choice.get("text"), str) + else choice + for choice in _dicts(response.get("choices")) + ) + + +def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + return _joined_choice( + tuple(text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None) + ) + + +def _transcription_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + text: Final = response.get("text") + return (_text_choice(text),) if isinstance(text, str) and text else () + + +def _moderation_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + return _joined_choice( + tuple( + _moderation_verdict(flagged, result.get("categories")) + for result in _dicts(response.get("results")) + if isinstance(flagged := result.get("flagged"), bool) + ) + ) + + +def _moderation_verdict(flagged: bool, categories: object) -> str: + if not flagged: + return "not flagged" + hits: Final = ( + tuple(name for name, hit in cast(Mapping[str, object], categories).items() if hit is True) + if isinstance(categories, dict) + else () + ) + return f"flagged: {', '.join(hits)}" if hits else "flagged" + + +def _rerank_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + return _joined_choice( + tuple( + _rerank_line(index, score, result.get("document")) + for result in _dicts(response.get("results")) + if (index := as_int(result.get("index"))) is not None + if (score := as_float(result.get("relevance_score"))) is not None + ) + ) + + +def _rerank_line(index: int, score: float, document: object) -> str: + text: Final = as_str((as_str_mapping(document) or {}).get("text")) + return f"[{index}] {score}\n{text}" if text else f"[{index}] {score}" + + +def _search_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + return _joined_choice( + tuple( + line + for result in _dicts(response.get("results")) + if (line := "\n".join(part for key in ("title", "url", "snippet") if (part := as_str(result.get(key))))) + ) + ) + + +def _image_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + return _joined_choice( + tuple(summary for item in _dicts(response.get("data")) if (summary := _image_summary(item)) is not None) + ) + + +def _image_summary(item: Mapping[str, object]) -> str | None: + location: Final = _image_location(item) + if location is None: + return None + revised: Final = as_str(item.get("revised_prompt")) + return f"{revised}\n{location}" if revised else location + + +def _image_location(item: Mapping[str, object]) -> str | None: + url: Final = as_str(item.get("url")) + if url is not None: + return url + encoded: Final = item.get("b64_json") + if not isinstance(encoded, str): + return None + return f"b64_json image ({len(encoded) * 3 // 4 - encoded[-2:].count('=')} bytes)" + + +def _binary_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]: + size: Final = as_int(response.get("num_bytes")) + if size is None: return () - message: Final[_AssistantMessage] = { - "role": "assistant", - "content": "\n\n".join(markdowns), - "refusal": None, - "tool_calls": None, - } - choice: Final[_Choice] = {"message": message, "finish_reason": None} - return (choice,) + content_type: Final = as_str(response.get("content_type")) + return (_text_choice(f"{content_type} ({size} bytes)" if content_type else f"{size} bytes"),) def _responses_parts_text(parts: tuple[Mapping[str, object], ...], part_type: str, field: str) -> str | None: diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index d3628005bac..f552ba37655 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -245,6 +245,7 @@ class GenAIEvent: details, unlike the deprecated ``error.message`` span attribute. """ + NAME_KEY: Final = "event.name" OPERATION_EXCEPTION: Final = "gen_ai.client.operation.exception" diff --git a/litellm/integrations/otel/plumbing/events.py b/litellm/integrations/otel/plumbing/events.py index e7b8e22ddcd..e95d31f886f 100644 --- a/litellm/integrations/otel/plumbing/events.py +++ b/litellm/integrations/otel/plumbing/events.py @@ -9,18 +9,28 @@ emitting that event; the exporter pipeline it rides is built in """ from dataclasses import dataclass +from time import time_ns from typing import Final -from opentelemetry._events import Event, EventLogger +from opentelemetry._logs import Logger, LogRecord from opentelemetry._logs.severity import SeverityNumber +from opentelemetry.sdk.resources import Resource from opentelemetry.trace import SpanContext from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent +try: + from opentelemetry.sdk._logs import LogRecord as _SDKLogRecord +except ImportError: + _SDKLogRecord = None + +SDK_LOG_RECORD: Final[type[LogRecord] | None] = _SDKLogRecord + @dataclass(frozen=True, slots=True) class GenAIEventRecorder: - event_logger: EventLogger + event_logger: Logger + resource: Resource | None = None def record_operation_exception( self, @@ -30,24 +40,36 @@ class GenAIEventRecorder: stack_trace: str | None, timestamp_ns: int | None, ) -> None: - # ``exception.type`` and ``exception.message`` are the semconv-required - # pair and always ride the event; only the recommended stacktrace is - # conditional on the payload carrying one. stacktrace: Final = ((ExceptionEvent.STACKTRACE, stack_trace),) if stack_trace else () - self.event_logger.emit( - Event( - name=GenAIEvent.OPERATION_EXCEPTION, - timestamp=timestamp_ns, + attributes: Final = dict( + ( + (GenAIEvent.NAME_KEY, GenAIEvent.OPERATION_EXCEPTION), + (ExceptionEvent.TYPE, error_type), + (ExceptionEvent.MESSAGE, message), + *stacktrace, + ) + ) + record: Final[LogRecord] = ( + SDK_LOG_RECORD( + timestamp=timestamp_ns or time_ns(), trace_id=span_context.trace_id, span_id=span_context.span_id, trace_flags=span_context.trace_flags, severity_number=SeverityNumber.WARN, - attributes=dict( - ( - (ExceptionEvent.TYPE, error_type), - (ExceptionEvent.MESSAGE, message), - *stacktrace, - ) - ), + body=message, + attributes=attributes, + resource=self.resource, # pyright: ignore[reportCallIssue] # SDK-only kwarg absent from the API LogRecord signature on the pin + ) + if SDK_LOG_RECORD is not None + else LogRecord( + timestamp=timestamp_ns or time_ns(), + trace_id=span_context.trace_id, + span_id=span_context.span_id, + trace_flags=span_context.trace_flags, + severity_number=SeverityNumber.WARN, + body=message, + attributes=attributes, + event_name=GenAIEvent.OPERATION_EXCEPTION, # pyright: ignore[reportCallIssue] # kwarg exists only on OTel 1.38+, absent from the pinned API signature ) ) + self.event_logger.emit(record) diff --git a/litellm/integrations/otel/plumbing/otlp_json.py b/litellm/integrations/otel/plumbing/otlp_json.py index b4b659f1e01..bc6d7d435b0 100644 --- a/litellm/integrations/otel/plumbing/otlp_json.py +++ b/litellm/integrations/otel/plumbing/otlp_json.py @@ -10,6 +10,7 @@ from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import Final, TypeAlias +import requests from google.protobuf.json_format import MessageToDict from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter @@ -62,8 +63,14 @@ def encode_spans_json(spans: Sequence[ReadableSpan]) -> bytes: class OTLPJsonSpanExporter(OTLPSpanExporter): - def __init__(self, endpoint: str | None, headers: dict[str, str]) -> None: # mutable-ok: SDK __init__ takes Dict - super().__init__(endpoint=endpoint, headers=headers) + def __init__( + self, + endpoint: str | None, + headers: dict[str, str], # mutable-ok: SDK __init__ takes Dict + certificate_file: str | None = None, + session: "requests.Session | None" = None, + ) -> None: + super().__init__(endpoint=endpoint, headers=headers, certificate_file=certificate_file, session=session) self._session.headers["Content-Type"] = JSON_CONTENT_TYPE def _serialize_spans(self, spans: Sequence[ReadableSpan]) -> bytes: diff --git a/litellm/integrations/otel/plumbing/otlp_tls.py b/litellm/integrations/otel/plumbing/otlp_tls.py new file mode 100644 index 00000000000..9de97e0c77a --- /dev/null +++ b/litellm/integrations/otel/plumbing/otlp_tls.py @@ -0,0 +1,41 @@ +import os +from dataclasses import dataclass +from typing import Final, Literal + +import requests +from requests.adapters import HTTPAdapter + + +@dataclass(frozen=True, slots=True) +class OtlpHttpTls: + certificate_file: str | None + session: requests.Session | None + + +class _NoVerifyAdapter(HTTPAdapter): + def cert_verify( + self, + conn: object, + url: str, + verify: bool | str, + cert: str | tuple[str, str] | None, + ) -> None: + super().cert_verify( # pyright: ignore[reportUnknownMemberType] # requests stubs omit HTTPAdapter.cert_verify + conn, url, False, cert + ) + + +def resolve_otlp_http_tls(signal: Literal["TRACES", "METRICS", "LOGS"]) -> OtlpHttpTls: + if os.getenv(f"OTEL_EXPORTER_OTLP_{signal}_CERTIFICATE") or os.getenv("OTEL_EXPORTER_OTLP_CERTIFICATE"): + return OtlpHttpTls(certificate_file=None, session=None) + + from litellm.llms.custom_httpx.http_handler import get_ssl_verify + + verify: Final = get_ssl_verify() + if verify is False: + session: Final = requests.Session() + session.mount("https://", _NoVerifyAdapter()) + return OtlpHttpTls(certificate_file=None, session=session) + if isinstance(verify, str): + return OtlpHttpTls(certificate_file=verify, session=None) + return OtlpHttpTls(certificate_file=None, session=None) diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 2c4375ce5f7..e3474edaf14 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -9,11 +9,9 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal from opentelemetry import _logs, baggage, metrics, trace -from opentelemetry._events import EventLogger -from opentelemetry._logs import LoggerProvider, NoOpLoggerProvider +from opentelemetry._logs import Logger, LoggerProvider, NoOpLoggerProvider from opentelemetry.context import Context from opentelemetry.metrics import MeterProvider, NoOpMeterProvider -from opentelemetry.sdk._events import EventLoggerProvider from opentelemetry.sdk._logs import LoggerProvider as SDKLoggerProvider from opentelemetry.sdk._logs.export import ( BatchLogRecordProcessor, @@ -58,6 +56,7 @@ from litellm.integrations.otel.plumbing.context import ( request_destinations, suppressed_backends, ) +from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls if TYPE_CHECKING: from opentelemetry.metrics import Meter @@ -193,18 +192,24 @@ def _exporter_from_spec(spec: ExporterSpec) -> SpanExporter: if kind in _OTLP_HTTP_JSON_KINDS: from litellm.integrations.otel.plumbing.otlp_json import OTLPJsonSpanExporter + tls: Final = resolve_otlp_http_tls("TRACES") return OTLPJsonSpanExporter( endpoint=spec.traces_endpoint or _otlp_traces_endpoint(spec.endpoint), headers=parse_headers(spec.headers), + certificate_file=tls.certificate_file, + session=tls.session, ) if kind in _OTLP_HTTP_KINDS: from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( OTLPSpanExporter as HTTPExporter, ) + http_tls: Final = resolve_otlp_http_tls("TRACES") return HTTPExporter( endpoint=spec.traces_endpoint or _otlp_traces_endpoint(spec.endpoint), headers=parse_headers(spec.headers), + certificate_file=http_tls.certificate_file, + session=http_tls.session, ) if kind in _OTLP_GRPC_KINDS: from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( @@ -902,9 +907,12 @@ def build_metric_reader(config: OpenTelemetryV2Config) -> "MetricReader": OTLPMetricExporter as HTTPMetricExporter, ) + tls: Final = resolve_otlp_http_tls("METRICS") exporter: Any = HTTPMetricExporter( endpoint=_otlp_metrics_endpoint(config.endpoint), headers=parse_headers(config.headers), + certificate_file=tls.certificate_file, + session=tls.session, ) elif kind in ("otlp_grpc", "grpc"): try: @@ -962,9 +970,12 @@ def build_log_exporter(config: OpenTelemetryV2Config) -> LogExporter: OTLPLogExporter as HTTPLogExporter, ) + tls: Final = resolve_otlp_http_tls("LOGS") return HTTPLogExporter( endpoint=_otlp_logs_endpoint(config.endpoint), headers=parse_headers(config.headers), + certificate_file=tls.certificate_file, + session=tls.session, ) if kind in ("otlp_grpc", "grpc"): try: @@ -1029,8 +1040,8 @@ def resolve_logger_provider( return provider -def get_event_logger(provider: SDKLoggerProvider, name: str = "litellm") -> EventLogger: - return EventLoggerProvider(logger_provider=provider).get_event_logger(name, litellm_version) +def get_event_logger(provider: SDKLoggerProvider, name: str = "litellm") -> Logger: + return provider.get_logger(name, litellm_version) def build_meter_provider( diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 37b7344917e..d62a6c3427a 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -10,6 +10,7 @@ import sys from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import replace from datetime import datetime, timedelta +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast from pydantic import BaseModel @@ -17,7 +18,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import print_verbose, verbose_logger -from litellm.constants import PROXY_LLM_PROVIDER_FALLBACK +from litellm.constants import PROXY_LLM_PROVIDER_FALLBACK, PROXY_REJECTED_BEFORE_ROUTING_KEY from litellm.exceptions import ( validate_rate_limit_category, validate_rate_limit_type, @@ -66,6 +67,7 @@ from litellm.types.proxy.carried_budget_state import ( from litellm.types.utils import ( StandardLoggingGuardrailInformation, StandardLoggingPayload, + StandardLoggingZeroCostDiagnostic, ) if TYPE_CHECKING: @@ -212,17 +214,22 @@ def _get_proxy_llm_router() -> Router | None: return llm_router -def _bounded_requested_model_label(requested_model: str | None, router_originated: bool = False) -> str | None: +def _bounded_requested_model_label(requested_model: object, router_originated: bool = False) -> str | None: """ Bound ``requested_model`` label cardinality: names the router recognizes (model names, deployment ids, aliases, routing groups, team public model names) or matches via a global or team wildcard/pattern route keep their own label value; any other client-supplied string collapses into the - single ``other`` bucket. With no proxy router to vouch for the string, - client-supplied values collapse to ``other`` while ``router_originated`` - values (emitted by an SDK ``Router``'s own deployment failure and - fallback events, where the proxy router never exists) pass through. + single ``other`` bucket, as does any non-string request ``model`` value. + With no proxy router to vouch for the string, client-supplied values + collapse to ``other`` while ``router_originated`` values (emitted by an + SDK ``Router``'s own deployment failure and fallback events, where the + proxy router never exists) pass through. """ + if requested_model is None: + return None + if not isinstance(requested_model, str): + return UNRECOGNIZED_REQUESTED_MODEL_LABEL if not requested_model: return requested_model llm_router: Final = _get_proxy_llm_router() @@ -713,6 +720,15 @@ class PrometheusLogger(CustomLogger): labelnames=self.get_labels_for_metric("litellm_requests_metric"), ) + self.litellm_zero_cost_requests_total = self._counter_factory( + name="litellm_zero_cost_requests_total", + documentation=( + "Requests that carried usage but were logged at $0 on a model whose pricing entry " + "has a non-zero rate, by reason (missing_pricing_key, pricing_not_applied, cost_calculation_error)" + ), + labelnames=self.get_labels_for_metric("litellm_zero_cost_requests_total"), + ) + # Cache metrics self.litellm_cache_hits_metric = self._counter_factory( name="litellm_cache_hits_metric", @@ -1410,6 +1426,11 @@ class PrometheusLogger(CustomLogger): enum_values=enum_values, label_context=label_context, ) + self._increment_zero_cost_requests_metric( + zero_cost_diagnostic=standard_logging_payload.get("zero_cost_diagnostic"), + enum_values=enum_values, + label_context=label_context, + ) # input, output, total token metrics self._increment_token_metrics( @@ -1983,6 +2004,30 @@ class PrometheusLogger(CustomLogger): amount=float(response_cost), ) + def _increment_zero_cost_requests_metric( + self, + zero_cost_diagnostic: StandardLoggingZeroCostDiagnostic | None, + enum_values: UserAPIKeyLabelValues, + label_context: PrometheusLabelFactoryContext, + ) -> None: + if zero_cost_diagnostic is None: + return + supported_labels: Final = self.get_labels_for_metric("litellm_zero_cost_requests_total") + reason_label: Final = ( + MappingProxyType({ZERO_COST_REASON_LABEL: zero_cost_diagnostic["reason"]}) + if ZERO_COST_REASON_LABEL in supported_labels + else MappingProxyType({}) + ) + labels: Final = MappingProxyType( + { + **prometheus_label_factory( + supported_enum_labels=supported_labels, enum_values=enum_values, label_context=label_context + ), + **reason_label, + } + ) + self.litellm_zero_cost_requests_total.labels(**labels).inc() + @staticmethod def _get_remaining_from_v3_rate_limit_headers( standard_logging_payload: StandardLoggingPayload | None, @@ -2333,6 +2378,8 @@ class PrometheusLogger(CustomLogger): team_alias=user_api_team_alias, user=user_id, model_id=standard_logging_payload.get("model_id", ""), + requested_model=standard_logging_payload.get("model_group"), + api_provider=standard_logging_payload.get("custom_llm_provider"), custom_metadata_labels=get_custom_labels_from_metadata( metadata=_get_combined_custom_metadata_from_standard_logging_payload( standard_logging_payload=standard_logging_payload @@ -2345,6 +2392,11 @@ class PrometheusLogger(CustomLogger): "litellm_llm_api_failed_requests_metric", enum_values, ) + self._increment_zero_cost_requests_metric( + zero_cost_diagnostic=standard_logging_payload.get("zero_cost_diagnostic"), + enum_values=enum_values, + label_context=PrometheusLabelFactoryContext(enum_values), + ) self.set_llm_deployment_failure_metrics(kwargs) await self._set_org_budget_metrics_after_api_request( org_id=user_api_key_org_id, @@ -2785,7 +2837,7 @@ class PrometheusLogger(CustomLogger): # On LiteLLM-side rejects (no deployment picked), route request_kwargs["model"] # into requested_model and leave deployment-scoped labels empty. - deployment_selected: Final = bool(model_id) + deployment_selected: Final = bool(model_id) and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY) if deployment_selected: label_litellm_model_name = litellm_model_name label_model_id = model_id diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 6a4c67c7db1..54578323fa4 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -10,6 +10,8 @@ import asyncio import math import uuid from collections.abc import AsyncIterator, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast from typing_extensions import Never, ReadOnly @@ -196,6 +198,46 @@ class _AcompletionNamedParams(TypedDict, total=False): _NO_ACREATE_NAMED: Final[_AcreateNamedParams] = {} _NO_ASEARCH_NAMED: Final[_AsearchNamedParams] = {} + + +def _as_str_mapping(value: object) -> Mapping[str, object] | None: + return value if isinstance(value, Mapping) else None # pyright: ignore[reportUnknownVariableType] # str-keyed request metadata is not narrowable from object + + +@dataclass(frozen=True, slots=True) +class _ParentRequestCorrelation: + """Correlation ids of the LLM request that triggered an intercepted search, so the search's + own spend log row and traces land under the same session/trace instead of a fresh one.""" + + session_id: str | None + trace_id: str | None + parent_request_id: str | None + parent_otel_span: object | None + + def as_search_metadata(self) -> Mapping[str, object]: + return MappingProxyType( + { + key: value + for key, value in ( + ("session_id", self.session_id), + ("trace_id", self.trace_id), + ("parent_request_id", self.parent_request_id), + ("litellm_parent_otel_span", self.parent_otel_span), + ) + if value is not None + } + ) + + def as_search_kwargs(self) -> Mapping[str, str]: + return MappingProxyType( + { + key: value + for key, value in (("litellm_session_id", self.session_id), ("litellm_trace_id", self.trace_id)) + if value is not None + } + ) + + _NO_ACOMPLETION_NAMED: Final[_AcompletionNamedParams] = {} @@ -1527,15 +1569,17 @@ class WebSearchInterceptionLogger(CustomLogger): "WebSearchInterception: Executing search for '%s' using provider '%s'", query, search_provider ) user_api_key_auth: Final = self._get_user_api_key_auth_from_kwargs(kwargs) + parent_correlation: Final = self._get_parent_request_correlation(kwargs, user_api_key_auth) search_metadata: Final = ( None if user_api_key_auth is None else self._build_search_request_metadata( user_api_key_auth=user_api_key_auth, search_tool_name=search_tool_name, + parent_correlation=parent_correlation, ) ) - search_kwargs: Final = { + configured_search_kwargs: Final = { key: value for key, value in search_litellm_params.items() if key != "search_provider" and value is not None @@ -1549,8 +1593,11 @@ class WebSearchInterceptionLogger(CustomLogger): if rich_queries: query_arg = rich_queries rich_objective = rich.get("objective") - if rich_objective and "objective" not in search_kwargs: - search_kwargs["objective"] = rich_objective + if rich_objective and "objective" not in configured_search_kwargs: + configured_search_kwargs["objective"] = rich_objective + search_kwargs: Final = MappingProxyType( + {**configured_search_kwargs, **parent_correlation.as_search_kwargs()} + ) result: Final = ( await litellm.asearch( query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs @@ -1624,6 +1671,7 @@ class WebSearchInterceptionLogger(CustomLogger): def _build_search_request_metadata( user_api_key_auth: "UserAPIKeyAuth", search_tool_name: str | None, + parent_correlation: _ParentRequestCorrelation, ) -> Mapping[str, object]: """ Spend-tracking metadata for the intercepted search, so its provider cost is logged @@ -1637,11 +1685,50 @@ class WebSearchInterceptionLogger(CustomLogger): ) return { # mutable-ok: litellm's metadata channel is a plain dict its logging path reads and enriches **user_api_key_metadata, + **parent_correlation.as_search_metadata(), "model_group": search_tool_name, "user_api_key": user_api_key_auth.api_key, "user_api_key_auth": user_api_key_auth, } + @staticmethod + def _get_parent_request_correlation( + kwargs: Mapping[str, object] | None, + user_api_key_auth: "UserAPIKeyAuth | None", + ) -> _ParentRequestCorrelation: + """Read the originating request's ids from the hook kwargs, which are either the raw call + kwargs (metadata/litellm_metadata at top level) or a logging payload (under litellm_params).""" + if not kwargs: + return _ParentRequestCorrelation(None, None, None, None) + litellm_params: Final = _as_str_mapping(kwargs.get("litellm_params")) + scopes: Final[tuple[Mapping[str, object], ...]] = ( + (kwargs,) if litellm_params is None else (kwargs, litellm_params) + ) + metadatas: Final[tuple[Mapping[str, object], ...]] = tuple( + metadata + for scope in scopes + for metadata_key in ("metadata", "litellm_metadata") + if (metadata := _as_str_mapping(scope.get(metadata_key))) is not None + ) + + def first_str(scope_key: str | None, metadata_key: str | None) -> str | None: + candidates: Final[tuple[object, ...]] = ( + *(scope.get(scope_key) for scope in scopes if scope_key is not None), + *(metadata.get(metadata_key) for metadata in metadatas if metadata_key is not None), + ) + return next((value for value in candidates if isinstance(value, str) and value), None) + + parent_otel_span: Final[object | None] = next( + (span for metadata in metadatas if (span := metadata.get("litellm_parent_otel_span")) is not None), + None if user_api_key_auth is None else user_api_key_auth.parent_otel_span, + ) + return _ParentRequestCorrelation( + session_id=first_str("litellm_session_id", "session_id"), + trace_id=first_str("litellm_trace_id", "trace_id"), + parent_request_id=first_str("litellm_call_id", None), + parent_otel_span=parent_otel_span, + ) + @staticmethod def _selected_search_tool_name(search_tool: Mapping[str, object] | None) -> str | None: if search_tool is None: diff --git a/litellm/litellm_core_utils/README.md b/litellm/litellm_core_utils/README.md index b61c8982762..a5f5e8326b9 100644 --- a/litellm/litellm_core_utils/README.md +++ b/litellm/litellm_core_utils/README.md @@ -6,8 +6,9 @@ Core files: - `streaming_handler.py`: The core streaming logic + streaming related helper utils - `core_helpers.py`: code used in `types/` - e.g. `map_finish_reason`. - `exception_mapping_utils.py`: utils for mapping exceptions to openai-compatible error types. -- `default_encoding.py`: code for loading the default encoding (tiktoken) +- `default_encoding.py`: code for loading the default Python tokenizer and bundled cache - `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name. - `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s" - `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion]) +Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Missing native bindings or unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer. diff --git a/litellm/litellm_core_utils/agentic_followup_kwargs.py b/litellm/litellm_core_utils/agentic_followup_kwargs.py new file mode 100644 index 00000000000..50ec19f62c4 --- /dev/null +++ b/litellm/litellm_core_utils/agentic_followup_kwargs.py @@ -0,0 +1,32 @@ +from collections.abc import Collection, Mapping, Sequence +from itertools import chain +from types import MappingProxyType +from typing import Final + + +def build_agentic_followup_kwargs( + *, + request_kwargs: Mapping[str, object], + patch_kwargs: Mapping[str, object], + request_params: Collection[str], + depth: int, + max_loops: int, + fingerprints: Sequence[str], + fingerprint: str, +) -> Mapping[str, object]: + """Kwargs for an agentic follow-up call: the request's kwargs overlaid by the plan's, never repeating a key already sent as a request param""" + seen: Final = [*fingerprints, fingerprint] # mutable-ok: the chat loop's settings reader only accepts a list + return MappingProxyType( + { + key: value + for key, value in chain( + ((k, v) for k, v in request_kwargs.items() if k not in request_params), + ((k, v) for k, v in patch_kwargs.items() if k not in request_params), + ( + ("_agentic_loop_depth", depth + 1), + ("max_agentic_loops", max_loops), + ("_agentic_loop_fingerprints", seen), + ), + ) + } + ) diff --git a/litellm/litellm_core_utils/bug_report.py b/litellm/litellm_core_utils/bug_report.py new file mode 100644 index 00000000000..25fddd6a10e --- /dev/null +++ b/litellm/litellm_core_utils/bug_report.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import os +import platform +import traceback +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal +from urllib.parse import urlencode + +import litellm +from litellm._version import version as litellm_version +from litellm.types.utils import LlmProviders + +ISSUE_URL_BASE: Final = "https://github.com/BerriAI/litellm/issues/new" +MAX_URL_LENGTH: Final = 6000 +MAX_FRAMES: Final = 12 +DISABLE_ENV_VAR: Final = "LITELLM_DISABLE_BUG_REPORT_LINK" +NOTICE_PREFIX: Final = "This looks like a bug in LiteLLM rather than in your request." +KNOWN_PROVIDERS: Final = frozenset(provider.value for provider in LlmProviders) + +Surface = Literal["sdk", "proxy"] + + +@dataclass(frozen=True, slots=True) +class EnvironmentReport: + surface: Surface + litellm_version: str + python_version: str + deployment: str | None + config_lines: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class BugReport: + environment: EnvironmentReport + exception_type: str + litellm_frames: tuple[str, ...] + call_type: str | None + custom_llm_provider: str | None + stream: bool | None + + +def bug_report_enabled() -> bool: + return os.getenv(DISABLE_ENV_VAR, "").lower() != "true" + + +def should_report_bug(exc: object) -> bool: + return bug_report_enabled() and isinstance(exc, BaseException) and getattr(exc, "status_code", None) is None + + +def _format_frame(frame: traceback.FrameSummary, package_dir: Path, package_parent: Path) -> str | None: + frame_path: Final = Path(frame.filename).resolve() + try: + frame_path.relative_to(package_dir) + relative_path: Final = frame_path.relative_to(package_parent) + except ValueError: + return None + return f"{relative_path.as_posix()}:{frame.lineno} in {frame.name}" + + +def _get_litellm_frames(exc: BaseException) -> tuple[str, ...]: + if exc.__traceback__ is None: + return () + package_dir: Final = Path(litellm.__file__).resolve().parent + package_parent: Final = package_dir.parent + return tuple( + frame_text + for frame in traceback.extract_tb(exc.__traceback__) + if (frame_text := _format_frame(frame, package_dir, package_parent)) is not None + )[-MAX_FRAMES:] + + +def allowlisted(value: object, allowed: frozenset[str]) -> str | None: + return value if isinstance(value, str) and value in allowed else None + + +def _deployment(surface: Surface) -> str | None: + if surface == "sdk": + return "pip / Python SDK" + return "Docker" if os.path.exists("/.dockerenv") else None + + +def build_environment_report(*, surface: Surface, config_lines: tuple[str, ...] = ()) -> EnvironmentReport: + return EnvironmentReport( + surface=surface, + litellm_version=litellm_version, + python_version=platform.python_version(), + deployment=_deployment(surface), + config_lines=config_lines, + ) + + +def build_bug_report( + exc: BaseException, + *, + surface: Surface, + call_type: str | None = None, + custom_llm_provider: object = None, + stream: object = None, + config_lines: tuple[str, ...] = (), +) -> BugReport: + return BugReport( + environment=build_environment_report(surface=surface, config_lines=config_lines), + exception_type=type(exc).__name__, + litellm_frames=_get_litellm_frames(exc), + call_type=call_type, + custom_llm_provider=allowlisted(custom_llm_provider, KNOWN_PROVIDERS), + stream=stream if isinstance(stream, bool) else None, + ) + + +def _domain(report: BugReport) -> str: + if report.environment.surface == "sdk": + return "Python SDK: the litellm package itself" + if report.custom_llm_provider is not None: + return "LLM translation: a specific provider's request or response" + return "Proxy core: startup, config, health checks, endpoints" + + +def _title(report: BugReport, frames: tuple[str, ...]) -> str: + location: Final = frames[-1].split(":", 1)[0] if frames else "litellm" + return f"[Bug]: {report.exception_type} in {location}" + + +def _description(report: BugReport, frames: tuple[str, ...], config_lines: tuple[str, ...]) -> str: + frame_block: Final = "LiteLLM frames:\n```\n" + "\n".join(frames) + "\n```\n\n" if frames else "" + stream_line: Final = "" if report.stream is None else f"Stream: {str(report.stream).lower()}\n" + config_block: Final = ( + "\nConfig (true/false flags and LiteLLM-defined values only):\n```\n" + "\n".join(config_lines) + "\n```\n" + if config_lines + else "" + ) + return ( + "Auto-generated by LiteLLM's bug report link. It carries no request data or error text. " + "Please describe what you were doing, and paste the error message from your log below " + "if it contains nothing sensitive.\n\n" + "```\n\n```\n\n" + f"Exception: `{report.exception_type}`\n\n" + f"{frame_block}" + f"Surface: {report.environment.surface}\n" + f"Endpoint / call: {report.call_type or 'unknown'}\n" + f"Provider: {report.custom_llm_provider or 'unknown'}\n" + f"LiteLLM: {report.environment.litellm_version}\n" + f"Python: {report.environment.python_version}\n" + f"{stream_line}" + f"{config_block}" + ) + + +def _issue_url(report: BugReport, frames: tuple[str, ...], config_lines: tuple[str, ...]) -> str: + deployment: Final[tuple[tuple[str, str], ...]] = ( + () if report.environment.deployment is None else (("deployment", report.environment.deployment),) + ) + fields: Final = ( + ("template", "bug_report.yml"), + ("labels", "bug"), + ("title", _title(report, frames)), + ("version", report.environment.litellm_version), + ("domain", _domain(report)), + ("description", _description(report, frames, config_lines)), + ) + deployment + return f"{ISSUE_URL_BASE}?{urlencode(fields)}" + + +def bug_report_issue_url(report: BugReport) -> str: + frames: Final = report.litellm_frames + config_lines: Final = report.environment.config_lines + candidates: Final = ( + *((frames, config_lines[:count]) for count in range(len(config_lines), -1, -1)), + *((frames[index:], ()) for index in range(1, len(frames) + 1)), + ) + return next( + ( + url + for candidate_frames, candidate_config in candidates + if len(url := _issue_url(report, candidate_frames, candidate_config)) <= MAX_URL_LENGTH + ), + _issue_url(report, (), ()), + ) + + +def strip_bug_report_notice(message: str) -> str: + index: Final = message.find(NOTICE_PREFIX) + if index == -1: + return message + head: Final = message[:index] + return head.removesuffix("\n") + + +def bug_report_notice(report: BugReport) -> str: + return ( + f"{NOTICE_PREFIX} File it with one click " + f"(prefilled, no request data or error text, review before submitting): {bug_report_issue_url(report)}" + ) diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index c8e9e2583ba..e0bd85a7937 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -2,10 +2,13 @@ import json from collections.abc import Mapping +from itertools import chain +from types import MappingProxyType from typing import Final, cast from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, @@ -117,13 +120,25 @@ def _wrap_response_as_fake_stream( ) -def _add_agentic_loop_metadata(kwargs_for_followup: dict[str, object]) -> None: - metadata = kwargs_for_followup.get("litellm_metadata") - metadata = dict(metadata) if isinstance(metadata, dict) else {} - for key, value in kwargs_for_followup.items(): - if key.startswith("_agentic_loop") or key == "max_agentic_loops" or is_interception_internal_key(key): - metadata[key] = value - kwargs_for_followup["litellm_metadata"] = metadata +def _with_agentic_loop_metadata(kwargs_for_followup: Mapping[str, object]) -> Mapping[str, object]: + metadata: Final = kwargs_for_followup.get("litellm_metadata") + return MappingProxyType( + { + **kwargs_for_followup, + "litellm_metadata": dict( # mutable-ok: the follow-up call's logging and proxy hooks write into litellm_metadata in place + chain( + metadata.items() if isinstance(metadata, dict) else (), + ( + (key, value) + for key, value in kwargs_for_followup.items() + if key.startswith("_agentic_loop") + or key == "max_agentic_loops" + or is_interception_internal_key(key) + ), + ) + ), + } + ) def _filter_followup_kwargs(source: dict[str, object]) -> dict[str, object]: @@ -165,14 +180,17 @@ async def _execute_chat_completion_agentic_plan( if "tool_choice" not in patch.optional_params: optional_params_for_followup.pop("tool_choice", None) - kwargs_for_followup: Final = _filter_followup_kwargs(kwargs) - kwargs_for_followup.update( - {k: v for k, v in _filter_followup_kwargs(patch.kwargs).items() if k not in optional_params_for_followup} + kwargs_for_followup: Final = _with_agentic_loop_metadata( + build_agentic_followup_kwargs( + request_kwargs=_filter_followup_kwargs(kwargs), + patch_kwargs=_filter_followup_kwargs(patch.kwargs), + request_params=frozenset((*optional_params_for_followup, "model", "messages")), + depth=depth, + max_loops=max_loops, + fingerprints=fingerprints, + fingerprint=fingerprint, + ) ) - kwargs_for_followup["_agentic_loop_depth"] = depth + 1 - kwargs_for_followup["max_agentic_loops"] = max_loops - kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint] - _add_agentic_loop_metadata(kwargs_for_followup) try: response_followup = await litellm.acompletion( diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index ce6f77f78a0..d7fbe9f7e09 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -3,7 +3,7 @@ import copy import logging import re -from collections.abc import Iterable, Mapping +from collections.abc import Collection, Iterable, Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol @@ -359,6 +359,46 @@ def get_litellm_metadata_from_kwargs(kwargs: dict): return {} +def _budget_reservation_on_auth_object(user_api_key_auth: object) -> object: + if isinstance(user_api_key_auth, Mapping): + return user_api_key_auth.get("budget_reservation") + return getattr(user_api_key_auth, "budget_reservation", None) + + +def budget_reservation_from_metadata(metadata: Mapping[str, object]) -> dict | None: + stamped: Final = metadata.get("user_api_key_budget_reservation") + if isinstance(stamped, dict): + return stamped + on_auth_object: Final = _budget_reservation_on_auth_object(metadata.get("user_api_key_auth")) + return on_auth_object if isinstance(on_auth_object, dict) else None + + +def _stamp_budget_reservation_callback_bound(litellm_params: Mapping[str, object], callback_bound: bool) -> None: + for metadata_variable_name in ("metadata", "litellm_metadata"): + metadata = litellm_params.get(metadata_variable_name) + if not isinstance(metadata, Mapping): + continue + budget_reservation = budget_reservation_from_metadata(metadata) + if budget_reservation is not None: + budget_reservation["callback_bound"] = callback_bound + + +def bind_budget_reservation_to_callbacks(litellm_params: Mapping[str, object]) -> None: + """Mark the request's budget reservation as owned by the success callbacks of this call. + + The proxy releases any reservation still unbound when the request ends; one bound here + is left for the cost callback, which may finish after the response has been sent. Bind + only where a success handler is guaranteed to run: a logging object merely existing is + not that, since the proxy builds one for every route before calling anything. + """ + _stamp_budget_reservation_callback_bound(litellm_params, True) + + +def unbind_budget_reservation_from_callbacks(litellm_params: Mapping[str, object]) -> None: + """Hand a failed call's reservation back to the request-end release: failure handlers never settle it.""" + _stamp_budget_reservation_callback_bound(litellm_params, False) + + def reconstruct_model_name( model_name: str, custom_llm_provider: str | None, @@ -669,10 +709,11 @@ def filter_internal_params(data: dict, additional_internal_params: set | None = def redact_nested_match_and_regex_keys( payload: dict | list[Any] | str | None, + keys: Collection[str] = ("match", "regex"), ) -> dict | list[Any] | str | None: """ - Deep-copy `payload` and replace every `match` / `regex` string field with - "[REDACTED]" anywhere in nested dict/list structures. + Deep-copy `payload` and replace every configured string field with "[REDACTED]" + anywhere in nested dict/list structures. Used for guardrail spend/compliance logging so raw spans are not persisted. """ @@ -694,10 +735,9 @@ def redact_nested_match_and_regex_keys( continue seen.add(node_id) if isinstance(node, dict): - if "match" in node: - node["match"] = "[REDACTED]" - if "regex" in node: - node["regex"] = "[REDACTED]" + for key in keys: + if key in node: + node[key] = "[REDACTED]" stack.extend(node.values()) elif isinstance(node, list): stack.extend(node) diff --git a/litellm/litellm_core_utils/default_encoding.py b/litellm/litellm_core_utils/default_encoding.py index 71b30614d8d..c3b6a008411 100644 --- a/litellm/litellm_core_utils/default_encoding.py +++ b/litellm/litellm_core_utils/default_encoding.py @@ -1,5 +1,4 @@ import os -from pathlib import Path from typing import Final import litellm @@ -15,20 +14,6 @@ except (ImportError, AttributeError): filename = pkg_resources.resource_filename(__name__, "litellm_core_utils/tokenizers") -CL100K_BASE_RANK_FILE: Final = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4" -O200K_BASE_RANK_FILE: Final = "fb374d419588a4632f3f557e76b4b70aebbca790" - - -def cl100k_base_rank_file() -> str: - """The vendored tiktoken `cl100k_base` rank file (`base64(token) rank` lines).""" - return Path(filename, CL100K_BASE_RANK_FILE).read_text(encoding="ascii") - - -def o200k_base_rank_file() -> str: - """The vendored tiktoken `o200k_base` rank file (`base64(token) rank` lines).""" - return Path(filename, O200K_BASE_RANK_FILE).read_text(encoding="ascii") - - # Always default TIKTOKEN_CACHE_DIR to the bundled tokenizers directory # unless the user explicitly overrides it via CUSTOM_TIKTOKEN_CACHE_DIR. # This keeps tiktoken fully offline-capable by default (see #1071). diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 425714d730f..da2f11f2593 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -10,6 +10,11 @@ import httpx import litellm from litellm._logging import _ENABLE_SECRET_REDACTION, _redact_string, verbose_logger +from litellm.litellm_core_utils.bug_report import ( + bug_report_notice, + build_bug_report, + should_report_bug, +) from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.types.utils import LlmProviders @@ -2673,7 +2678,21 @@ def exception_type( ) else: raise APIConnectionError( - message=f"{original_exception}\n{_redact_string(traceback.format_exc())}", + message=( + f"{original_exception}\n{_redact_string(traceback.format_exc())}" + + ( + "\n" + + bug_report_notice( + build_bug_report( + original_exception, + surface="sdk", + custom_llm_provider=custom_llm_provider, + ) + ) + if should_report_bug(original_exception) + else "" + ) + ), llm_provider=custom_llm_provider, model=model, request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), # stub the request diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 9b2db9aad18..112d038039e 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -4,6 +4,7 @@ from typing import Final from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.llms.openai.data_residency import infer_openai_data_residency +from litellm.types.router import CustomPricingLiteLLMParams AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset( { @@ -23,10 +24,7 @@ AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset( } ) -# Keys `completion()` forwards from its own kwargs into `get_litellm_params`, -# which are otherwise invisible to it because that call site passes explicit -# named arguments rather than `**kwargs`. -FORWARDED_KWARGS_KEYS: Final = AWS_CREDENTIAL_KWARGS_KEYS +PROVIDER_AFFINITY_HEADER_KWARG_KEY: Final = "provider_affinity_header" # Pre-define optional kwargs keys as frozenset for O(1) lookups # These are extracted from kwargs only if present, avoiding unnecessary .get() calls @@ -46,6 +44,8 @@ OPTIONAL_KWARGS_KEYS: Final = ( "bucket_name", "s3_endpoint_url", "s3_region_name", + "s3_access_key_id", + "s3_secret_access_key", "vertex_credentials", "vertex_project", "vertex_location", @@ -60,9 +60,11 @@ OPTIONAL_KWARGS_KEYS: Final = ( "itpm", "otpm", "use_xai_oauth", + PROVIDER_AFFINITY_HEADER_KWARG_KEY, } ) | AWS_CREDENTIAL_KWARGS_KEYS + | frozenset(CustomPricingLiteLLMParams.model_fields) ) # Backward-compatible alias for existing imports/tests. diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 5868e79323a..f622098920c 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -859,6 +859,8 @@ def _get_openai_compatible_provider_info( elif custom_llm_provider == "edenai": api_base = litellm.EdenAIChatConfig.get_api_base(api_base) # rebind-ok: chain resolves in place dynamic_api_key = litellm.EdenAIChatConfig.get_api_key(api_key) # rebind-ok: chain resolves in place + elif custom_llm_provider == "fal_ai": + dynamic_api_key = litellm.FalAIChatConfig.get_api_key(api_key) # rebind-ok: chain resolves in place elif custom_llm_provider == "aiml": ( api_base, diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 91a22144805..5471fe50d5f 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -17,14 +17,16 @@ import random import sys import threading import time -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass, replace from datetime import datetime, timezone from importlib.resources import files from pathlib import Path +from types import MappingProxyType from typing import Final, Protocol import httpx +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm import verbose_logger @@ -37,6 +39,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" +_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) _CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"}) @@ -88,6 +91,18 @@ class GetModelCostMap: """Load the local backup model cost map bundled with the package.""" return GetModelCostMap.load_local_model_cost_map_with_revision().model_cost_map + _loaded_catalog: Mapping[str, Mapping[str, object]] = MappingProxyType({}) + + @classmethod + def loaded_model_cost_map(cls) -> Mapping[str, Mapping[str, object]]: + """The catalog as last loaded (bundled or remote), untouched by ``register_model`` or router registrations.""" + return cls._loaded_catalog + + @classmethod + def _snapshot_loaded_catalog(cls, model_cost: Mapping[str, object]) -> None: + raw: Final = _CATALOG_ADAPTER.validate_python(model_cost) + cls._loaded_catalog = MappingProxyType({key: MappingProxyType(entry) for key, entry in raw.items()}) + @classmethod def _get_backup_model_count(cls) -> int: """Return the number of models in the local backup (cached int).""" @@ -533,7 +548,9 @@ def _finalize_model_cost_map(model_cost: dict) -> dict: def _finalize_loaded_model_cost_map(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded: _cost_map_source_info.source_revision = loaded.revision _cost_map_source_info.etag = loaded.etag - return replace(loaded, model_cost_map=_finalize_model_cost_map(loaded.model_cost_map)) + finalized: Final = _finalize_model_cost_map(loaded.model_cost_map) + GetModelCostMap._snapshot_loaded_catalog(finalized) # pyright: ignore[reportPrivateUsage] # same module + return replace(loaded, model_cost_map=finalized) def adopt_model_cost_map( diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index e4744079622..00ab05aba77 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -1,10 +1,24 @@ import re -from collections.abc import Iterator, Mapping +from collections.abc import Generator, Iterator, Mapping +from contextlib import contextmanager +from contextvars import ContextVar from typing import Any, Final from litellm.types.utils import OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams _CLIENT_CALLBACK_METADATA_SLOTS: Final[tuple[str, ...]] = ("litellm_metadata", "metadata") +_inherited_message_logging_disabled: Final[ContextVar[bool]] = ContextVar( + "inherited_message_logging_disabled", default=False +) + + +@contextmanager +def inherit_message_logging_privacy(disabled: bool) -> Generator[None]: + token: Final = _inherited_message_logging_disabled.set(_inherited_message_logging_disabled.get() or disabled) + try: + yield + finally: + _inherited_message_logging_disabled.reset(token) def iter_client_callback_metadata_dicts( @@ -84,6 +98,8 @@ _supported_callback_params: Final[tuple[str, ...]] = ( "arize_api_key", "arize_space_key", "arize_space_id", + "arize_success_sampling_rate", + "arize_error_sampling_rate", "posthog_api_key", "posthog_host", "braintrust_api_key", @@ -143,7 +159,7 @@ def get_trusted_callback_params(kwargs: Mapping[str, Any] | None) -> tuple[tuple def initialize_standard_callback_dynamic_params( - kwargs: dict | None = None, + kwargs: dict[str, object] | None = None, ) -> StandardCallbackDynamicParams: """ Initialize the standard callback dynamic params from the kwargs @@ -179,4 +195,10 @@ def initialize_standard_callback_dynamic_params( if param in _trusted_overlay_callback_params: standard_callback_dynamic_params[param] = trusted_value + if _inherited_message_logging_disabled.get(): + private_params: Final[StandardCallbackDynamicParams] = { + **standard_callback_dynamic_params, + "turn_off_message_logging": True, + } + return private_params return standard_callback_dynamic_params diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index b34f1b3aafd..6e5a37a7226 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -50,6 +50,8 @@ from litellm.cost_calculator import ( RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, _select_model_name_for_cost_calc, + get_usage_object, + pricing_entry_for_cost_calc, ) from litellm.exceptions import ( BudgetExceededError, @@ -89,6 +91,10 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ( InteractionsUsageObjectTransformation, ) +from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import ( + diagnose_zero_cost, + zero_cost_warning, +) from litellm.litellm_core_utils.logging_utils import ( truncate_base64_in_messages, truncate_base64_in_messages_async, @@ -101,6 +107,10 @@ from litellm.litellm_core_utils.redact_messages import ( redact_streaming_responses_for_custom_logger, should_redact_message_logging, ) +from litellm.litellm_core_utils.served_output_texts import ( + SERVED_OUTPUT_TEXTS_KEY, + overlay_served_output_texts, +) from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.responses.utils import ResponseAPILoggingUtils @@ -157,6 +167,7 @@ from litellm.types.utils import ( StandardLoggingPayloadStatusFields, StandardLoggingPromptManagementMetadata, StandardLoggingVectorStoreRequest, + StandardLoggingZeroCostDiagnostic, TextCompletionResponse, TranscriptionResponse, Usage, @@ -210,7 +221,7 @@ from .initialize_dynamic_callback_params import ( from .specialty_caches.dynamic_logging_cache import DynamicLoggingCache if TYPE_CHECKING: - from mcp.types import EmbeddedResource, ImageContent, TextContent + from mcp.types import CallToolResult, EmbeddedResource, ImageContent, TextContent from litellm.integrations.otel.logger import OpenTelemetryV2 from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config @@ -376,11 +387,40 @@ _DEPLOYMENT_PRICING_KEYS: Final = ( "output_cost_per_token", "input_cost_per_token_batches", "output_cost_per_token_batches", + "input_cost_per_token_above_272k_tokens_batches", + "output_cost_per_token_above_272k_tokens_batches", + "cache_read_input_token_cost_batches", + "cache_read_input_token_cost_above_272k_tokens_batches", + "cache_creation_input_token_cost_batches", + "cache_creation_input_token_cost_above_272k_tokens_batches", "ocr_cost_per_page", "ocr_cost_per_page_batches", "annotation_cost_per_page", "annotation_cost_per_page_batches", ) +_INPUT_PRICING_KEY_PREFIXES: Final = ( + "input_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", +) +_OUTPUT_PRICING_KEY_PREFIXES: Final = ("output_cost_per_token",) +_BATCH_PRICING_KEY_SUFFIX: Final = "_batches" + + +_NO_CARRIED_RATES: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _published_direction( + published: ModelInfo, registered: Mapping[str, object], flat_key: str, prefixes: tuple[str, ...] +) -> Mapping[str, object]: + return MappingProxyType( + { + key: value + for key, value in published.items() + if registered.get(key) is None + and (key == flat_key or (key.startswith(prefixes) and key.endswith(_BATCH_PRICING_KEY_SUFFIX))) + } + ) def deployment_pricing_model_info(model_id: str | None, deployment_model: str | None) -> ModelInfo | None: @@ -392,12 +432,15 @@ def deployment_pricing_model_info(model_id: str | None, deployment_model: str | get_model_info fills absent costs with 0, so asking it directly cannot tell "configured as free" apart from "no pricing configured". A deployment may declare only one side of its pricing, so the side it leaves out keeps - the model's published rates instead of billing as zero. Ownership is per - token direction: declaring either rate for a direction takes that whole - direction, so a published batch rate can never displace a standard rate - the deployment configured itself. OCR per-page rates count as declared - pricing too; they pass through as registered and ``ocr_batch_cost`` layers - the published rate under each per-page family the deployment leaves out. + the model's published standard rate and every published batch rate for + that direction (flat, long-context tier, cached, cache write) instead of + billing as zero. Ownership is per token direction: declaring the flat + standard or flat batch rate for a direction takes that whole direction, so + a published batch rate can never displace a standard rate the deployment + configured itself. A tier-only override keeps every published rate it left + out. OCR per-page rates count as declared pricing too; they pass through as + registered and ``ocr_batch_cost`` layers the published rate under each + per-page family the deployment leaves out. """ if model_id is None: return None @@ -418,13 +461,22 @@ def deployment_pricing_model_info(model_id: str | None, deployment_model: str | registered.get("output_cost_per_token") is not None or registered.get("output_cost_per_token_batches") is not None ) - if not declares_input: - merged["input_cost_per_token"] = published.get("input_cost_per_token") - merged["input_cost_per_token_batches"] = published.get("input_cost_per_token_batches") - if not declares_output: - merged["output_cost_per_token"] = published.get("output_cost_per_token") - merged["output_cost_per_token_batches"] = published.get("output_cost_per_token_batches") - return merged + carried_input: Final = ( + _NO_CARRIED_RATES + if declares_input + else _published_direction(published, registered, "input_cost_per_token", _INPUT_PRICING_KEY_PREFIXES) + ) + carried_output: Final = ( + _NO_CARRIED_RATES + if declares_output + else _published_direction(published, registered, "output_cost_per_token", _OUTPUT_PRICING_KEY_PREFIXES) + ) + priced: Final[ModelInfo] = { # pyright: ignore[reportAssignmentType] # carried keys are ModelInfo rates + **merged, + **carried_input, + **carried_output, + } + return priced def _published_pricing(deployment_model: str | None) -> ModelInfo | None: @@ -489,6 +541,14 @@ def mask_api_base_credentials(api_base: str) -> str: return api_base[:key_end] + "*" * 5 + api_base[-4:] +def _timestamp_seconds(moment: object) -> float | None: + if isinstance(moment, datetime.datetime): + return moment.timestamp() + if isinstance(moment, (int, float)): + return float(moment) + return None + + class Logging(LiteLLMLoggingBaseClass): global \ supabaseClient, \ @@ -614,6 +674,7 @@ class Logging(LiteLLMLoggingBaseClass): self.truncated_messages_for_logging: str | list | dict | None = None # mutable-ok: logged messages shape ## TIME TO FIRST TOKEN LOGGING ## self.completion_start_time: datetime.datetime | None = None + self.zero_cost_warned: bool = False self._llm_caching_handler: LLMCachingHandler | None = None # INITIAL LITELLM_PARAMS @@ -1573,15 +1634,11 @@ class Logging(LiteLLMLoggingBaseClass): async def async_post_mcp_tool_call_hook( self, kwargs: dict, - response_obj: Any, + response_obj: "CallToolResult", start_time: datetime.datetime, end_time: datetime.datetime, - ): - """ - Post MCP Tool Call Hook - - Use this to modify the MCP tool call response before it is returned to the user. - """ + ) -> "CallToolResult": + """Apply ordered MCP content callbacks to the result returned to the caller.""" from litellm.types.llms.base import HiddenParams from litellm.types.mcp import MCPPostCallResponseObject @@ -1589,24 +1646,51 @@ class Logging(LiteLLMLoggingBaseClass): dynamic_success_callbacks=self.dynamic_success_callbacks, global_callbacks=litellm.success_callback, ) - post_mcp_tool_call_response_obj: Final[MCPPostCallResponseObject] = MCPPostCallResponseObject( - mcp_tool_call_response=response_obj, hidden_params=HiddenParams() - ) + hidden_params = HiddenParams() for callback in callbacks: try: if isinstance(callback, CustomLogger): - response: MCPPostCallResponseObject | None = await callback.async_post_mcp_tool_call_hook( - kwargs=kwargs, - response_obj=post_mcp_tool_call_response_obj, - start_time=start_time, - end_time=end_time, + original_content = copy.deepcopy(response_obj.content) + original_structured_content = copy.deepcopy(response_obj.structured_content) + callback_response = MCPPostCallResponseObject( + mcp_tool_call_response=copy.deepcopy(original_content), hidden_params=hidden_params ) - ###################################################################### - # if any of the callbacks modify the response, use the modified response - # current implementation returns the first modified response - ###################################################################### - if response is not None: - response_obj = self._parse_post_mcp_call_hook_response(response=response) + try: + response = await callback.async_post_mcp_tool_call_hook( + kwargs=kwargs, + response_obj=callback_response, + start_time=start_time, + end_time=end_time, + ) + hook_content = ( + self._parse_post_mcp_call_hook_response(response=response) + if response is not None + else callback_response.mcp_tool_call_response + ) + if response is not None: + hidden_params = response.hidden_params + except Exception as e: + verbose_logger.exception( + "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e + ) + hook_content = None + structured_replacement_matches = ( + response_obj.structured_content != original_structured_content + and ( + hook_content is None + or hook_content == original_content + or response_obj.content == hook_content + ) + ) + if hook_content is not None and hook_content != original_content: + response_obj.content[:] = hook_content + if ( + response_obj.content != original_content + and response_obj.structured_content is not None + and not structured_replacement_matches + ): + response_obj.structured_content = None + response_obj.is_error = True except Exception as e: verbose_logger.exception("LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e) return response_obj @@ -1626,10 +1710,12 @@ class Logging(LiteLLMLoggingBaseClass): return response.mcp_tool_call_response def get_response_ms(self) -> float: - return ( - self.model_call_details.get("end_time", datetime.datetime.now()) - - self.model_call_details.get("start_time", datetime.datetime.now()) - ).total_seconds() * 1000 + now: Final = datetime.datetime.now() + start_seconds: Final = _timestamp_seconds(self.model_call_details.get("start_time", now)) + end_seconds: Final = _timestamp_seconds(self.model_call_details.get("end_time", now)) + if start_seconds is None or end_seconds is None: + return 0.0 + return (end_seconds - start_seconds) * 1000 def set_cost_breakdown( self, @@ -1757,17 +1843,26 @@ class Logging(LiteLLMLoggingBaseClass): if transformed_result is not None: result = transformed_result - result_hidden_params: Final = getattr(result, "_hidden_params", None) or MappingProxyType({}) - result_additional_headers: Final = ( - result_hidden_params.get("additional_headers") - if isinstance(result_hidden_params, dict) - else getattr(result_hidden_params, "additional_headers", None) + priced_result: Final = ( + result.response + if isinstance(result, (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent)) + else result ) - if isinstance(result, (BaseModel, HttpxBinaryResponseContent)) and hasattr(result, "_hidden_params"): + + result_hidden_params: Final = getattr(priced_result, "_hidden_params", None) or MappingProxyType({}) + if isinstance(priced_result, (BaseModel, HttpxBinaryResponseContent)) and hasattr( + priced_result, "_hidden_params" + ): hidden_params: Final = result_hidden_params if ( "response_cost" in hidden_params and hidden_params["response_cost"] is not None ): # use cost if already calculated + self._record_zero_cost_diagnostic( + priced_result, + hidden_params["response_cost"], + litellm_model_name=litellm_model_name, + router_model_id=router_model_id or hidden_params.get("model_id"), + ) return hidden_params["response_cost"] elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set router_model_id = hidden_params["model_id"] @@ -1779,18 +1874,7 @@ class Logging(LiteLLMLoggingBaseClass): router_model_id = self.get_router_model_id() ## RESPONSE COST ## - spilled_over: Final = is_spilled_over_ptu_request( - model_info=_deployment_model_info(self.litellm_params if hasattr(self, "litellm_params") else None), - response_headers=self.model_call_details.get("response_headers"), - additional_headers=result_additional_headers, - ) - custom_pricing: Final = ( - False - if spilled_over - else use_custom_pricing_for_model( - litellm_params=(self.litellm_params if hasattr(self, "litellm_params") else None) - ) - ) + custom_pricing: Final = self._custom_pricing_for(priced_result) prompt = self._prompt_for_cost_calculation() @@ -1799,7 +1883,7 @@ class Logging(LiteLLMLoggingBaseClass): try: response_cost_calculator_kwargs: Final = { - "response_object": result, + "response_object": priced_result, "model": litellm_model_name or self.model, "cache_hit": cache_hit, "custom_llm_provider": self.model_call_details.get("custom_llm_provider", None), @@ -1842,9 +1926,18 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.debug("response_cost: %s", response_cost) additional_response_cost: Final[object] = self.model_call_details.get("additional_response_cost") - if isinstance(additional_response_cost, (int, float)) and additional_response_cost > 0: - return (response_cost or 0.0) + additional_response_cost - return response_cost + total_response_cost: Final = ( + (response_cost or 0.0) + additional_response_cost + if isinstance(additional_response_cost, (int, float)) and additional_response_cost > 0 + else response_cost + ) + self._record_zero_cost_diagnostic( + priced_result, + total_response_cost, + litellm_model_name=litellm_model_name, + router_model_id=router_model_id, + ) + return total_response_cost except Exception as e: # error calculating cost debug_info = StandardLoggingModelCostFailureDebugInformation( error_str=str(e), @@ -1858,9 +1951,108 @@ class Logging(LiteLLMLoggingBaseClass): ) verbose_logger.debug("response_cost_failure_debug_information: %s", debug_info) self.model_call_details["response_cost_failure_debug_information"] = debug_info + self._record_zero_cost_diagnostic( + priced_result, + None, + calculation_failed=True, + litellm_model_name=litellm_model_name, + router_model_id=router_model_id, + ) return None + def _record_zero_cost_diagnostic( + self, + result: object, + response_cost: float | None, + *, + calculation_failed: bool = False, + litellm_model_name: str | None = None, + router_model_id: str | None = None, + ) -> None: + if response_cost is None and not calculation_failed: + return + if self.model_call_details.get("cache_hit") is True: + self.model_call_details["zero_cost_diagnostic"] = None + return + try: + finding: Final = self._zero_cost_finding( + result, + response_cost, + calculation_failed=calculation_failed, + litellm_model_name=litellm_model_name, + router_model_id=router_model_id, + ) + except Exception as e: # noqa: BLE001 # the pricing helpers raise plain Exception and a diagnostic must never break cost tracking + verbose_logger.debug("zero_cost_diagnostic skipped: %s", e) + return + self.model_call_details["zero_cost_diagnostic"] = finding[0] if finding is not None else None + if finding is None or self.zero_cost_warned: + return + self.zero_cost_warned = True + verbose_logger.warning(finding[1]) + + def _zero_cost_finding( + self, + result: object, + response_cost: float | None, + *, + calculation_failed: bool, + litellm_model_name: str | None, + router_model_id: str | None, + ) -> tuple[StandardLoggingZeroCostDiagnostic, str] | None: + metadata: Final = StandardLoggingPayloadSetup.merge_litellm_metadata(self.litellm_params) + if response_cost or is_unbilled_non_inference_call(self.call_type, metadata, result): + return None + usage: Final = get_usage_object(completion_response=result) + if usage is None: + return None + model: Final = litellm_model_name or self.model + custom_llm_provider: Final = self.model_call_details.get("custom_llm_provider") + pricing: Final = pricing_entry_for_cost_calc( + model=model, + completion_response=result, + custom_llm_provider=custom_llm_provider, + custom_pricing=self._custom_pricing_for(result), + base_model=_get_base_model_from_metadata(model_call_details=self.model_call_details), + router_model_id=router_model_id or self.get_router_model_id(), + region_name=_resolve_mantle_region_for_cost( + custom_llm_provider=custom_llm_provider, + litellm_params=self.model_call_details.get("litellm_params"), + ), + litellm_logging_obj=self, + ) + if pricing is None: + return None + diagnostic: Final = diagnose_zero_cost( + usage=usage, pricing_model=pricing[0], pricing_entry=pricing[1], calculation_failed=calculation_failed + ) + if diagnostic is None: + return None + model_group: Final = metadata.get("model_group") + return diagnostic, zero_cost_warning( + diagnostic, + model_group=model_group if isinstance(model_group, str) else None, + model=model, + custom_llm_provider=custom_llm_provider, + usage=usage, + ) + + def _custom_pricing_for(self, result: object) -> bool: + litellm_params: Final = getattr(self, "litellm_params", None) + result_hidden_params: Final = getattr(result, "_hidden_params", None) or MappingProxyType({}) + additional_headers: Final = ( + result_hidden_params.get("additional_headers") + if isinstance(result_hidden_params, dict) + else getattr(result_hidden_params, "additional_headers", None) + ) + spilled_over: Final = is_spilled_over_ptu_request( + model_info=_deployment_model_info(litellm_params), + response_headers=self.model_call_details.get("response_headers"), + additional_headers=additional_headers, + ) + return False if spilled_over else use_custom_pricing_for_model(litellm_params=litellm_params) + def _prompt_for_cost_calculation(self) -> str: """ The raw input string is only priced directly for text-to-speech, which bills per character. @@ -2205,6 +2397,7 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["response_cost"] = 0.0 elif "response_cost" in hidden_params: self.model_call_details["response_cost"] = hidden_params["response_cost"] + self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"]) elif (existing_cost := self.model_call_details.get("response_cost")) is not None and existing_cost != 0: # Preserve response_cost if already calculated (e.g., by pass-through # handlers like Gemini/Vertex which call completion_cost directly). @@ -5378,7 +5571,7 @@ def request_model_access_groups_from_litellm_params(litellm_params: Mapping[str, """Access groups the auth layer stamped onto this request, from whichever metadata field carries them. Detached internal sub-calls only inherit the identity keys, so the auth object is the - fallback there, exactly as _get_budget_reservation_from_metadata does for reservations. + fallback there, exactly as budget_reservation_from_metadata does for reservations. """ for metadata_variable_name in ("metadata", "litellm_metadata"): metadata = litellm_params.get(metadata_variable_name) @@ -5718,15 +5911,12 @@ class StandardLoggingPayloadSetup: modified_final_response_obj: Final = redact_message_input_output_from_logging( model_call_details=kwargs, - result=final_response_obj, + result=overlay_served_output_texts(final_response_obj, kwargs.get(SERVED_OUTPUT_TEXTS_KEY)), ) if modified_final_response_obj is not None and isinstance(modified_final_response_obj, BaseModel): - final_response_obj = modified_final_response_obj.model_dump() - else: - final_response_obj = modified_final_response_obj - - return final_response_obj + return modified_final_response_obj.model_dump() + return modified_final_response_obj @staticmethod def get_additional_headers( @@ -6162,6 +6352,8 @@ def _extract_response_obj_and_hidden_params( hidden_params = getattr(init_response_obj, "_hidden_params", None) elif isinstance(init_response_obj, dict): response_obj = init_response_obj + elif isinstance(init_response_obj, HttpxBinaryResponseContent): + response_obj = dict(init_response_obj.logging_summary()) else: response_obj = {} @@ -6499,6 +6691,7 @@ def get_standard_logging_object_payload( error_str=error_str, error_information=error_information, response_cost_failure_debug_info=kwargs.get("response_cost_failure_debug_information"), + zero_cost_diagnostic=kwargs.get("zero_cost_diagnostic"), guardrail_information=metadata.get("standard_logging_guardrail_information", None), standard_built_in_tools_params=standard_built_in_tools_params, ) @@ -6677,6 +6870,7 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload: response_cost=response_cost, autorouter_savings=None, response_cost_failure_debug_info=None, + zero_cost_diagnostic=None, status="success", total_tokens=int(DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT), prompt_tokens=int(DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT), diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index c60e3089816..46bf2ec2960 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -24,6 +24,7 @@ from litellm.types.utils import ( CallTypes, CompletionTokensDetailsWrapper, CostPerToken, + CustomPricingLiteLLMParams, DataResidency, ImageResponse, ModelInfo, @@ -49,6 +50,15 @@ _IMAGE_RESPONSE_CALL_TYPES: Final = frozenset( # Pre-resolved DataResidency enum values for fast membership checks _VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency) +_DEPLOYMENT_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields) + +_IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = ( + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_image_token", + "output_cost_per_image_token", +) + # Pre-resolved service-tier cost-key suffixes (e.g. "_priority"). Used per # request in the cost-calc path, so the f-strings are built once here instead # of being rebuilt for every model_info key on every call. Longest-first so a @@ -67,6 +77,15 @@ _SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType( ) _INCLUSIVE_THRESHOLD_PROVIDERS: Final = frozenset({"xai"}) +_BATCH_KEY_SUFFIX: Final = "_batches" +_BATCH_RATE_PREFIXES: Final = ( + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", +) +_BATCH_TIER_KEY: Final = re.compile(rf"^({'|'.join(_BATCH_RATE_PREFIXES)})_above_(\d+k?)_tokens{_BATCH_KEY_SUFFIX}$") +_NON_STANDARD_THRESHOLD_SUFFIXES: Final = (*_SERVICE_TIER_SUFFIXES, _BATCH_KEY_SUFFIX) def _uses_inclusive_token_thresholds(custom_llm_provider: str | None) -> bool: @@ -248,9 +267,80 @@ def _get_service_tier_cost_key(base_key: str, service_tier: str | None) -> str: return f"{base_key}_{suffix}" +def _parse_token_threshold(threshold: str) -> float: + return float(threshold.replace("k", "")) * (1000 if "k" in threshold else 1) + + def _parse_above_token_threshold(key: str) -> float: - threshold_str: Final = key.split("_above_")[1].split("_tokens")[0] - return float(threshold_str.replace("k", "")) * (1000 if "k" in threshold_str else 1) + return _parse_token_threshold(key.split("_above_")[1].split("_tokens")[0]) + + +def _prompt_exceeds_threshold(prompt_tokens: int, threshold: float, inclusive: bool) -> bool: + return prompt_tokens > threshold or (inclusive and prompt_tokens == threshold) + + +@dataclass(frozen=True, slots=True) +class BatchCostRates: + input: float | None + output: float | None + cache_read: float | None + cache_creation: float | None + + +def _batch_rate(model_info: ModelInfo, key: str) -> float | None: + value: Final = model_info.get(key) + if isinstance(value, (int, float)): + return float(value) + if not isinstance(value, str): + return None + try: + return float(value) + except ValueError: + return None + + +def _batch_tier_rate(model_info: ModelInfo, tier_key: str, flat_key: str) -> float | None: + tier_rate: Final = _batch_rate(model_info, tier_key) + return _batch_rate(model_info, flat_key) if tier_rate is None else tier_rate + + +def _batch_tier_thresholds(model_info: ModelInfo, prefix: str) -> frozenset[str]: + return frozenset( + tier.group(2) + for key, value in model_info.items() + if value is not None and (tier := _BATCH_TIER_KEY.match(key)) is not None and tier.group(1) == prefix + ) + + +def _crossed_batch_tier(model_info: ModelInfo, prefix: str, usage: Usage, inclusive: bool) -> str | None: + return next( + ( + threshold + for threshold in sorted( + _batch_tier_thresholds(model_info, prefix), key=_parse_token_threshold, reverse=True + ) + if _prompt_exceeds_threshold(usage.prompt_tokens, _parse_token_threshold(threshold), inclusive) + ), + None, + ) + + +def _batch_rate_for_prefix(model_info: ModelInfo, prefix: str, usage: Usage, inclusive: bool) -> float | None: + flat_key: Final = f"{prefix}{_BATCH_KEY_SUFFIX}" + threshold: Final = _crossed_batch_tier(model_info, prefix, usage, inclusive) + if threshold is None: + return _batch_rate(model_info, flat_key) + return _batch_tier_rate(model_info, f"{prefix}_above_{threshold}_tokens{_BATCH_KEY_SUFFIX}", flat_key) + + +def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provider: str | None) -> BatchCostRates: + inclusive: Final = _uses_inclusive_token_thresholds(custom_llm_provider) + return BatchCostRates( + input=_batch_rate_for_prefix(model_info, "input_cost_per_token", usage, inclusive), + output=_batch_rate_for_prefix(model_info, "output_cost_per_token", usage, inclusive), + cache_read=_batch_rate_for_prefix(model_info, "cache_read_input_token_cost", usage, inclusive), + cache_creation=_batch_rate_for_prefix(model_info, "cache_creation_input_token_cost", usage, inclusive), + ) def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict | None: @@ -576,7 +666,9 @@ def _get_token_base_cost( # so that the threshold detection loop only processes standard keys. The # service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key. threshold_keys: Final = [ - k for k in model_info if k.startswith("input_cost_per_token_above_") and not k.endswith(_SERVICE_TIER_SUFFIXES) + k + for k in model_info + if k.startswith("input_cost_per_token_above_") and not k.endswith(_NON_STANDARD_THRESHOLD_SUFFIXES) ] # Only sort the threshold keys (typically 1-2 keys instead of 66+) @@ -588,7 +680,7 @@ def _get_token_base_cost( # Handle both formats: _above_128k_tokens and _above_128_tokens threshold_str = key.split("_above_")[1].split("_tokens")[0] threshold = _parse_above_token_threshold(key) - if usage.prompt_tokens > threshold or (threshold_is_inclusive and usage.prompt_tokens == threshold): + if _prompt_exceeds_threshold(usage.prompt_tokens, threshold, threshold_is_inclusive): # Prefer a service_tier-specific above-threshold key when available, # e.g. input_cost_per_token_priority_above_200k_tokens for Gemini # ON_DEMAND_PRIORITY. Falls back to the standard key automatically @@ -744,6 +836,53 @@ def _get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: floa return default_value +def deployment_pricing(model_info: ModelInfo | None) -> ModelInfo | None: + """The prices a deployment sets itself, as floats; None when it sets none that parse.""" + if model_info is None: + return None + priced_keys: Final = tuple(key for key in _DEPLOYMENT_PRICING_KEYS if model_info.get(key) is not None) + pricing: Final = MappingProxyType( + { + key: price + for key in priced_keys + if (price := _get_cost_per_unit(model_info, key, default_value=None)) is not None + } + ) + if not pricing: + return None + return cast(ModelInfo, pricing) # cast-ok: a read-only subset of ModelInfo pricing keys, values validated above + + +def prices_tokens(model_info: ModelInfo) -> bool: + """Whether the price table carries any token rate, so a token-priced calculator can bill from usage.""" + return any(model_info.get(key) is not None for key in _IMAGE_TOKEN_RATE_KEYS) + + +def flat_image_cost(model_info: ModelInfo | None, image_response: ImageResponse) -> float: + """The per-image price times the images returned; 0.0 when the table sets no per-image price.""" + if model_info is None: + return 0.0 + output_cost_per_image: Final = _get_cost_per_unit(model_info, "output_cost_per_image", default_value=None) or 0.0 + num_images: Final = len(image_response.data) if image_response.data else 0 + return output_cost_per_image * num_images + + +def resolve_image_model_info(model: str, custom_llm_provider: str, model_info: ModelInfo | None) -> ModelInfo: + """The price table an image cost calculator consults for ``model``. + + ``shared_backend_model_info`` keeps deployment prices off the shared ``{provider}/{model}`` key, so + a name lookup alone reads the public rate, and a model only the deployment prices has no entry at all. + """ + if model_info is None: + return get_model_info(model=model, custom_llm_provider=custom_llm_provider) + try: + shared_model_info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model + return model_info + resolved: Final[ModelInfo] = {**shared_model_info, **model_info} + return resolved + + def calculate_cache_writing_cost( cache_creation_tokens: int, cache_creation_token_details: CacheCreationTokenDetails | None, @@ -1629,6 +1768,7 @@ def calculate_image_response_cost_from_usage( model: str, image_response: ImageResponse, custom_llm_provider: str, + model_info: ModelInfo | None = None, ) -> float | None: """ Calculate image generation cost from usage metadata when available. @@ -1653,6 +1793,9 @@ def calculate_image_response_cost_from_usage( if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0: return None + if model_info is not None and not prices_tokens(model_info): + return None + input_tokens_details: Final[object] = getattr(usage, "input_tokens_details", None) prompt_tokens_details: PromptTokensDetailsWrapper | None = None if input_tokens_details is not None: @@ -1708,6 +1851,7 @@ def calculate_image_response_cost_from_usage( model=model, usage=normalized_usage, custom_llm_provider=custom_llm_provider, + model_info=model_info, ) return prompt_cost + completion_cost @@ -1768,9 +1912,15 @@ class CostCalculatorUtils: size: str | None = None, optional_params: dict | None = None, call_type: str | None = None, + model_info: ModelInfo | None = None, ) -> float: """ Route the image generation cost calculator based on the custom_llm_provider + + ``model_info`` is the deployment's own price table. Its valid prices are laid over the shared + cost-map entry and handed to the provider calculator, so per-image, per-pixel and per-token + deployment prices all apply while provider logic (token-first billing, grounding surcharges, + image counting) stays in one place. An unparseable price is logged and ignored. """ from litellm.cost_calculator import default_image_cost_calculator from litellm.llms.azure_ai.image_generation.cost_calculator import ( @@ -1796,12 +1946,14 @@ class CostCalculatorUtils: quality or completion_response.quality or _requested_image_param(optional_params, "quality") or "standard" ) resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0) + pricing: Final = deployment_pricing(model_info) if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value: if isinstance(completion_response, ImageResponse): return vertex_ai_image_cost_calculator( model=model, image_response=completion_response, + model_info=pricing, ) elif custom_llm_provider == litellm.LlmProviders.BEDROCK.value: if isinstance(completion_response, ImageResponse): @@ -1820,6 +1972,7 @@ class CostCalculatorUtils: return recraft_image_cost_calculator( model=model, image_response=completion_response, + model_info=pricing, ) elif custom_llm_provider == litellm.LlmProviders.AIML.value: from litellm.llms.aiml.image_generation.cost_calculator import ( @@ -1829,6 +1982,7 @@ class CostCalculatorUtils: return aiml_image_cost_calculator( model=model, image_response=completion_response, + model_info=pricing, ) elif custom_llm_provider == litellm.LlmProviders.COMETAPI.value: from litellm.llms.cometapi.image_generation.cost_calculator import ( @@ -1838,6 +1992,7 @@ class CostCalculatorUtils: return cometapi_image_cost_calculator( model=model, image_response=completion_response, + model_info=pricing, ) elif custom_llm_provider == litellm.LlmProviders.GEMINI.value: if call_type in ( @@ -1851,6 +2006,7 @@ class CostCalculatorUtils: return gemini_image_edit_cost_calculator( model=model, image_response=completion_response, + model_info=pricing, ) from litellm.llms.gemini.image_generation.cost_calculator import ( cost_calculator as gemini_image_cost_calculator, @@ -1859,6 +2015,7 @@ class CostCalculatorUtils: return gemini_image_cost_calculator( model=model, image_response=completion_response, + model_info=pricing, ) elif custom_llm_provider == litellm.LlmProviders.AZURE_AI.value: return azure_ai_image_cost_calculator( @@ -1867,6 +2024,7 @@ class CostCalculatorUtils: size=resolved_size, n=resolved_n, optional_params=optional_params, + model_info=pricing, ) elif custom_llm_provider == litellm.LlmProviders.FAL_AI.value: from litellm.llms.fal_ai.cost_calculator import ( @@ -1877,6 +2035,7 @@ class CostCalculatorUtils: model=model, image_response=completion_response, optional_params=optional_params, + model_info=pricing, ) elif custom_llm_provider == litellm.LlmProviders.RUNWAYML.value: from litellm.llms.runwayml.cost_calculator import ( @@ -1886,6 +2045,7 @@ class CostCalculatorUtils: return runwayml_image_cost_calculator( model=model, image_response=completion_response, + model_info=pricing, ) elif ( custom_llm_provider == litellm.LlmProviders.OPENAI.value @@ -1902,6 +2062,7 @@ class CostCalculatorUtils: model=model, image_response=completion_response, custom_llm_provider=custom_llm_provider, + model_info=pricing, ) # Fall through to default for DALL-E models return default_image_cost_calculator( @@ -1911,6 +2072,7 @@ class CostCalculatorUtils: n=resolved_n, size=resolved_size, optional_params=optional_params, + model_info=pricing, ) else: return default_image_cost_calculator( @@ -1920,5 +2082,6 @@ class CostCalculatorUtils: n=resolved_n, size=resolved_size, optional_params=optional_params, + model_info=pricing, ) return 0.0 diff --git a/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py b/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py new file mode 100644 index 00000000000..6331d815bdc --- /dev/null +++ b/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py @@ -0,0 +1,146 @@ +from collections.abc import Mapping +from functools import reduce +from typing import Final + +from pydantic import TypeAdapter, ValidationError +from typing_extensions import assert_never + +from litellm.types.utils import StandardLoggingZeroCostDiagnostic, Usage + +ZERO_COST_COUNTER_NAME: Final = "litellm_zero_cost_requests_total" + +_TEXT_INPUT_RATE: Final = "input_cost_per_token" +_AUDIO_INPUT_RATE: Final = "input_cost_per_audio_token" +_TEXT_OUTPUT_RATE: Final = "output_cost_per_token" +_AUDIO_OUTPUT_RATE: Final = "output_cost_per_audio_token" +_RATE_KEY_MARKERS: Final = ("cost", "pricing") +_NESTED_PRICING: Final = TypeAdapter(Mapping[str, object] | tuple[object, ...]) +_MAX_PRICING_DEPTH: Final = 4 + + +def _audio_tokens(details: object) -> int: + audio_tokens: Final = getattr(details, "audio_tokens", None) + return audio_tokens if isinstance(audio_tokens, int) and audio_tokens > 0 else 0 + + +def _tokens(value: object) -> int: + return value if isinstance(value, int) and value > 0 else 0 + + +def used_pricing_keys(usage: Usage) -> tuple[str, ...]: + prompt_audio: Final = _audio_tokens(usage.prompt_tokens_details) + completion_audio: Final = _audio_tokens(usage.completion_tokens_details) + prompt_text: Final = _tokens(usage.prompt_tokens) - prompt_audio + completion_text: Final = _tokens(usage.completion_tokens) - completion_audio + components: Final = ( + (_TEXT_INPUT_RATE, prompt_text), + (_AUDIO_INPUT_RATE, prompt_audio), + (_TEXT_OUTPUT_RATE, completion_text), + (_AUDIO_OUTPUT_RATE, completion_audio), + ) + return tuple(key for key, count in components if count > 0) + + +def _nested_pricing(value: object) -> Mapping[str, object] | tuple[object, ...] | None: + try: + return _NESTED_PRICING.validate_python(value) + except ValidationError: + return None + + +def _is_rate_key(key: str) -> bool: + return any(marker in key for marker in _RATE_KEY_MARKERS) + + +def _rate_values(value: object) -> tuple[object, ...]: + nested: Final = _nested_pricing(value) + if isinstance(nested, Mapping): + return tuple(child for key, child in nested.items() if _is_rate_key(key)) + if nested is None: + return (value,) + return nested + + +def _expand_rate_values(values: tuple[object, ...], _depth: int) -> tuple[object, ...]: + return tuple(nested for value in values for nested in _rate_values(value)) + + +def _is_positive_number(value: object) -> bool: + return not isinstance(value, bool) and isinstance(value, (int, float)) and value > 0 + + +def _declares_a_rate(pricing_entry: Mapping[str, object]) -> bool: + leaves: Final = reduce(_expand_rate_values, range(_MAX_PRICING_DEPTH), (pricing_entry,)) + return any(_is_positive_number(leaf) for leaf in leaves) + + +def _is_explicit_zero(value: object) -> bool: + return not isinstance(value, bool) and isinstance(value, (int, float)) and value == 0 + + +def diagnose_zero_cost( + usage: Usage, + pricing_model: str, + pricing_entry: Mapping[str, object], + calculation_failed: bool, +) -> StandardLoggingZeroCostDiagnostic | None: + used_keys: Final = used_pricing_keys(usage) + if not used_keys: + return None + missing_keys: Final = tuple(key for key in used_keys if pricing_entry.get(key) is None) + if not missing_keys and all(_is_explicit_zero(pricing_entry[key]) for key in used_keys): + return None + if not _declares_a_rate(pricing_entry): + return None + if calculation_failed: + return StandardLoggingZeroCostDiagnostic( + reason="cost_calculation_error", pricing_model=pricing_model, missing_pricing_keys=() + ) + if missing_keys: + return StandardLoggingZeroCostDiagnostic( + reason="missing_pricing_key", pricing_model=pricing_model, missing_pricing_keys=missing_keys + ) + return StandardLoggingZeroCostDiagnostic( + reason="pricing_not_applied", pricing_model=pricing_model, missing_pricing_keys=() + ) + + +def _cause(diagnostic: StandardLoggingZeroCostDiagnostic) -> str: + reason: Final = diagnostic["reason"] + match reason: + case "missing_pricing_key": + return ( + f"pricing entry '{diagnostic['pricing_model']}' has no {', '.join(diagnostic['missing_pricing_keys'])}. " + "Set the missing rate in the deployment's model_info or in the model cost map, " + "or set every rate to 0 to mark the model free" + ) + case "pricing_not_applied": + return ( + f"pricing entry '{diagnostic['pricing_model']}' declares non-zero rates for this usage, " + "but the cost calculator returned $0" + ) + case "cost_calculation_error": + return ( + f"cost calculation raised for pricing entry '{diagnostic['pricing_model']}', " + "see response_cost_failure_debug_information" + ) + case _: + return assert_never(reason) + + +def zero_cost_warning( + diagnostic: StandardLoggingZeroCostDiagnostic, + *, + model_group: str | None, + model: str, + custom_llm_provider: str | None, + usage: Usage, +) -> str: + request: Final = ( + f"model_group={model_group or model} model={model} provider={custom_llm_provider or 'unknown'} " + f"prompt_tokens={_tokens(usage.prompt_tokens)} completion_tokens={_tokens(usage.completion_tokens)}" + ) + return ( + f"Billable request priced at $0 and logged as such ({request}): {_cause(diagnostic)}. " + f'Counted in {ZERO_COST_COUNTER_NAME}{{reason="{diagnostic["reason"]}"}}' + ) diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 4a3c8de78c5..503814cc143 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -251,6 +251,6 @@ def update_response_metadata( return metadata: Final = ResponseMetadata(result) - metadata.set_hidden_params(logging_obj, model, kwargs) metadata.set_timing_metrics(start_time, end_time, logging_obj, include_overhead) + metadata.set_hidden_params(logging_obj, model, kwargs) metadata.apply() diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 4ba7c3966c0..1e96e20a03b 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2261,6 +2261,37 @@ def system_messages_first( ] +def _system_content_as_text_parts(content: object) -> tuple[object, ...]: + if isinstance(content, str): + return (ChatCompletionTextObject(type="text", text=content),) + return tuple(cast(Sequence[object], content)) # cast-ok: non-str system content is a list of content parts + + +def _merge_system_message_run(run: Sequence[AllMessageValues]) -> AllMessageValues: + if len(run) == 1: + return run[0] + contents: Final = tuple(content for content in (message.get("content") for message in run) if content is not None) + if not contents: + return run[0] + if all(isinstance(content, str) for content in contents): + joined_text: Final = "\n\n".join(cast(tuple[str, ...], contents)) # cast-ok: every content is a str + return cast(AllMessageValues, {**run[0], "content": joined_text}) # cast-ok: dict spread keeps message shape + merged_parts: Final = [ # mutable-ok: chat message content must stay a json list + part for content in contents for part in _system_content_as_text_parts(content) + ] + return cast(AllMessageValues, {**run[0], "content": merged_parts}) # cast-ok: dict spread keeps message shape + + +def merge_consecutive_system_messages( + messages: list[AllMessageValues], # mutable-ok: message pipelines type messages as mutable lists +) -> list[AllMessageValues]: # mutable-ok: message pipelines type messages as mutable lists + return [ # mutable-ok: pipelines mutate message lists + merged + for is_system_run, run in groupby(messages, key=lambda message: message.get("role") == "system") + for merged in ((_merge_system_message_run(tuple(run)),) if is_system_run else run) + ] + + def _attempt_json_repair(s: str) -> object | None: """ Attempt to repair truncated JSON produced by LLM tool calls. diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index c9933422cc3..c44c80bc0a0 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -82,7 +82,7 @@ def _rejected_image_fetch(url: str, verdict: SSRFError) -> "litellm.ImageFetchEr verbose_logger.warning("Image fetch of %s rejected before any request went out: %s", url, verdict) return litellm.ImageFetchError( "Error: Unable to fetch image from URL. The proxy could not resolve this host or its URL policy rejected it; " - f"an admin can check the proxy log and `user_url_allowed_hosts` in general_settings. url={url}" + f"an admin can check the proxy log and `user_url_allowed_hosts` in litellm_settings. url={url}" ) diff --git a/litellm/litellm_core_utils/provider_affinity.py b/litellm/litellm_core_utils/provider_affinity.py new file mode 100644 index 00000000000..31cd9a7ff69 --- /dev/null +++ b/litellm/litellm_core_utils/provider_affinity.py @@ -0,0 +1,98 @@ +import re +from collections.abc import Mapping +from typing import Final + +from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY + +_HTTP_HEADER_NAME_PATTERN: Final = re.compile(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$") +_FORBIDDEN_AFFINITY_HEADERS: Final = frozenset( + { + "api-key", + "authorization", + "connection", + "content-length", + "content-type", + "cookie", + "host", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "set-cookie", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "www-authenticate", + "x-api-key", + "x-goog-api-key", + } +) + + +def validate_provider_affinity_header_name(header: str) -> str: + if not _HTTP_HEADER_NAME_PATTERN.fullmatch(header): + raise ValueError("provider_affinity_header must be a valid HTTP header name") + if header.lower() in _FORBIDDEN_AFFINITY_HEADERS: + raise ValueError("provider_affinity_header cannot be an authentication, cookie, or transport header") + return header + + +def _get_value(value: object, key: str) -> object | None: + if isinstance(value, Mapping): + return value.get(key) + return getattr(value, key, None) + + +def _get_provider_affinity_header_name(litellm_params: object | None) -> str | None: + header: Final = _get_value(litellm_params, "provider_affinity_header") if litellm_params is not None else None + if header is None: + return None + if not isinstance(header, str): + raise TypeError("provider_affinity_header must be a string") + return validate_provider_affinity_header_name(header) + + +def get_stable_session_id(litellm_params: object | None) -> str | None: + if litellm_params is None: + return None + + direct_session_id: Final = _get_value(litellm_params, "session_id") + if direct_session_id: + return str(direct_session_id) + + metadata_values: Final[tuple[object, ...]] = tuple( + value for key in ("metadata", "litellm_metadata") if (value := _get_value(litellm_params, key)) is not None + ) + has_generated_session_id: Final = any( + isinstance(metadata, Mapping) and metadata.get(SESSION_ID_GENERATED_METADATA_KEY) + for metadata in metadata_values + ) + + litellm_session_id: Final = _get_value(litellm_params, "litellm_session_id") + if litellm_session_id and not has_generated_session_id: + return str(litellm_session_id) + + for metadata in metadata_values: + if ( + isinstance(metadata, Mapping) + and not metadata.get(SESSION_ID_GENERATED_METADATA_KEY) + and (value := metadata.get("session_id")) + ): + return str(value) + return None + + +def add_provider_affinity_header( # mutable-ok: downstream handlers add auth and signing headers + headers: Mapping[str, object], litellm_params: object | None +) -> dict[str, object]: # mutable-ok: downstream handlers add auth and signing headers + header_name: Final = _get_provider_affinity_header_name(litellm_params) + if header_name is None or any(key.lower() == header_name.lower() for key in headers): + return dict(headers) # mutable-ok: downstream handlers add auth and signing headers + + session_id: Final = get_stable_session_id(litellm_params) + if session_id is None: + return dict(headers) # mutable-ok: downstream handlers add auth and signing headers + if any(character in session_id for character in ("\r", "\n", "\0")): + raise ValueError("session_id cannot contain HTTP header control characters") + return {**headers, header_name: session_id} # mutable-ok: downstream handlers add auth and signing headers diff --git a/litellm/litellm_core_utils/realtime_errors.py b/litellm/litellm_core_utils/realtime_errors.py index 3c064728a66..07f7148815c 100644 --- a/litellm/litellm_core_utils/realtime_errors.py +++ b/litellm/litellm_core_utils/realtime_errors.py @@ -9,13 +9,20 @@ frame itself fail, which is how a loud failure turns back into a silent one. """ import json -from typing import Final +from types import MappingProxyType +from typing import Final, Protocol from litellm.types.realtime import RealtimeErrorDetail, RealtimeErrorEvent WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123 +class _ClientWebSocket(Protocol): + async def send_text(self, data: str) -> None: ... + + async def close(self, code: int = ..., reason: str | None = ...) -> None: ... + + def realtime_error_event(message: str, error_type: str) -> str: detail: Final[RealtimeErrorDetail] = {"type": error_type, "message": message} event: Final[RealtimeErrorEvent] = {"type": "error", "error": detail} @@ -37,3 +44,28 @@ def client_close_code(upstream_code: int) -> int: if upstream_code in EXTERNAL_CLOSE_CODES or 3000 <= upstream_code < 5000: return upstream_code return int(CloseCode.INTERNAL_ERROR) + + +def upstream_handshake_close_code(status_code: int) -> int: + from websockets.frames import CloseCode + + refusal_codes: Final = MappingProxyType( + { + 401: int(CloseCode.POLICY_VIOLATION), + 403: int(CloseCode.POLICY_VIOLATION), + 429: int(CloseCode.TRY_AGAIN_LATER), + } + ) + return refusal_codes.get(status_code, int(CloseCode.INTERNAL_ERROR)) + + +async def close_after_upstream_handshake_refusal(websocket: _ClientWebSocket, status_code: int) -> None: + message: Final = f"Upstream realtime handshake rejected with HTTP {status_code}" + try: + await websocket.send_text(realtime_error_event(message, error_type="server_error")) + except Exception: # noqa: BLE001 # best-effort notice: a dead client socket must not skip the close below + pass + await websocket.close( + code=upstream_handshake_close_code(status_code), + reason=websocket_close_reason(message, fallback="Upstream handshake rejected"), + ) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index b409b181a79..8c77ef32cfb 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.classifier_logging import without_classifier_aud from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, ) +from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY from litellm.llms.vertex_ai.common_utils import ( redact_vertex_ai_metadata_from_litellm_params, redact_vertex_ai_metadata_from_logged_object, @@ -245,6 +246,8 @@ def _redact_model_response_dict_choices(choices, redacted_str: str): if "audio" in choice["delta"]: choice["delta"]["audio"] = None _redact_tool_calls_dict(choice["delta"]) + elif choice.get("text") is not None: + choice["text"] = redacted_str else: _redact_choice_content(choice) @@ -265,6 +268,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}] model_call_details["prompt"] = "" model_call_details["input"] = "" + model_call_details.pop(SERVED_OUTPUT_TEXTS_KEY, None) standard_logging_object: Final = model_call_details.get("standard_logging_object") if isinstance(standard_logging_object, Mapping): model_call_details["standard_logging_object"] = _redact_standard_logging_object(standard_logging_object) diff --git a/litellm/litellm_core_utils/served_output_texts.py b/litellm/litellm_core_utils/served_output_texts.py new file mode 100644 index 00000000000..323561a3dd2 --- /dev/null +++ b/litellm/litellm_core_utils/served_output_texts.py @@ -0,0 +1,171 @@ +"""Assistant text the caller received, per choice, so the logging payload stores the response a +post-call guardrail rewrote rather than the provider response the proxy assembled before it ran.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Final, Literal + +from pydantic import BaseModel, TypeAdapter, ValidationError + +from litellm.types.utils import ModelResponse, ModelResponseStream + +SERVED_OUTPUT_TEXTS_KEY: Final = "served_output_texts" + +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) +_JSON_LIST: Final = TypeAdapter(list[object]) +_TEXTS: Final = TypeAdapter(tuple[str | None, ...]) + +ServedTexts = tuple[str | None, ...] + + +class _TextBlock(BaseModel): + type: str + text: str | None = None + + +class _AnthropicMessage(BaseModel): + type: Literal["message"] + content: list[_TextBlock] + + +class _ResponsesOutputItem(BaseModel): + type: str + content: list[_TextBlock] = [] + + +class _ResponsesResponse(BaseModel): + object: Literal["response"] + output: list[_ResponsesOutputItem] + + +class _ChatChoices(BaseModel): + choices: list[object] + + +def _as_json_object(response: object) -> dict[str, object] | None: + candidate: Final = response.model_dump() if isinstance(response, BaseModel) else response + try: + return _JSON_OBJECT.validate_python(candidate) + except ValidationError: + return None + + +def _joined_block_texts(blocks: Sequence[_TextBlock], *, text_type: str) -> str | None: + texts: Final = tuple(block.text for block in blocks if block.type == text_type and block.text is not None) + return "".join(texts) if texts else None + + +def _chat_texts(response: ModelResponse) -> ServedTexts | None: + texts: Final = tuple( + choice.message.content if isinstance(choice.message.content, str) else None for choice in response.choices + ) + return texts if any(text is not None for text in texts) else None + + +def _anthropic_message_text(response: dict[str, object]) -> str | None: + try: + message: Final = _AnthropicMessage.model_validate(response) + except ValidationError: + return None + return _joined_block_texts(message.content, text_type="text") + + +def _responses_api_text(response: dict[str, object]) -> str | None: + try: + parsed: Final = _ResponsesResponse.model_validate(response) + except ValidationError: + return None + texts: Final = tuple( + text + for item in parsed.output + if item.type == "message" and (text := _joined_block_texts(item.content, text_type="output_text")) is not None + ) + return "".join(texts) if texts else None + + +def _chat_dict_texts(response: dict[str, object]) -> ServedTexts | None: + try: + _ChatChoices.model_validate(response) + return _chat_texts(ModelResponse(**response)) + except (ValidationError, TypeError, ValueError): + return None + + +def served_output_texts(response: object) -> ServedTexts | None: + if isinstance(response, ModelResponse): + return _chat_texts(response) + mapping: Final = _as_json_object(response) + if mapping is None: + return None + chat_texts: Final = _chat_dict_texts(mapping) + if chat_texts is not None: + return chat_texts + anthropic_text: Final = _anthropic_message_text(mapping) + text: Final = anthropic_text if anthropic_text is not None else _responses_api_text(mapping) + return (text,) if text is not None else None + + +def served_stream_output_texts(chunks: Sequence[object]) -> ServedTexts | None: + if chunks and all(isinstance(chunk, ModelResponseStream) for chunk in chunks): + return _chat_stream_texts(tuple(chunk for chunk in chunks if isinstance(chunk, ModelResponseStream))) + from litellm.proxy.guardrails.anthropic_sse import assemble_anthropic_sse_stream, is_anthropic_sse_stream + + if not is_anthropic_sse_stream(chunks): + return None + assembled: Final = assemble_anthropic_sse_stream(chunks) + return _chat_texts(assembled) if assembled is not None else None + + +def _chat_stream_choice_text(chunks: Sequence[ModelResponseStream], index: int) -> str | None: + contents: Final = tuple( + content + for chunk in chunks + for choice in chunk.choices + if choice.index == index and isinstance(content := choice.delta.content, str) + ) + return "".join(contents) if contents else None + + +def _chat_stream_texts(chunks: Sequence[ModelResponseStream]) -> ServedTexts | None: + choice_count: Final = max((choice.index + 1 for chunk in chunks for choice in chunk.choices), default=0) + texts: Final = tuple(_chat_stream_choice_text(chunks, index) for index in range(choice_count)) + return texts if any(text is not None for text in texts) else None + + +def record_served_output_texts(model_call_details: dict[str, object], texts: ServedTexts | None) -> None: + if texts is None: + return + model_call_details[SERVED_OUTPUT_TEXTS_KEY] = texts # rebind-ok: model_call_details is the shared kwargs bag + + +def overlay_served_output_texts( + response_obj: dict[str, object] | str | list[object] | None, served_texts: object +) -> dict[str, object] | str | list[object] | None: + if not isinstance(response_obj, dict): + return response_obj + logged: Final = _as_json_object(response_obj) + if logged is None: + return response_obj + try: + texts: Final = _TEXTS.validate_python(served_texts) + choices: Final = _JSON_LIST.validate_python(logged.get("choices")) + except ValidationError: + return response_obj + return { + **logged, + "choices": [ + _choice_with_text(choice, texts[index]) if index < len(texts) else choice + for index, choice in enumerate(choices) + ], + } + + +def _choice_with_text(choice: object, text: str | None) -> object: + choice_obj: Final = _as_json_object(choice) + if choice_obj is None or text is None: + return choice + message: Final = _as_json_object(choice_obj.get("message")) + if message is None or message.get("content") == text: + return choice + return {**choice_obj, "message": {**message, "content": text}} diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 025db65a7ce..aa4e0cf5495 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -165,6 +165,14 @@ class _UsageSummary(TypedDict): cost: float | None +def _reports_prompt_side_usage(usage_summary: "_UsageSummary") -> bool: + return ( + (usage_summary["prompt_tokens"] or 0) > 0 + or (usage_summary["cache_creation_input_tokens"] or 0) > 0 + or (usage_summary["cache_read_input_tokens"] or 0) > 0 + ) + + def capture_cache_creation_token_details( prompt_tokens_details: PromptTokensDetailsWrapper | None, current: CacheCreationTokenDetails | None, @@ -888,11 +896,11 @@ class ChunkProcessor: if usage_chunk_dict["completion_tokens"] is not None and usage_chunk_dict["completion_tokens"] > 0: completion_usage_updates += 1 if usage_chunk_dict["cache_creation_input_tokens"] is not None and ( - usage_chunk_dict["cache_creation_input_tokens"] > 0 or cache_creation_input_tokens is None + _reports_prompt_side_usage(usage_chunk_dict) or cache_creation_input_tokens is None ): cache_creation_input_tokens = usage_chunk_dict["cache_creation_input_tokens"] if usage_chunk_dict["cache_read_input_tokens"] is not None and ( - usage_chunk_dict["cache_read_input_tokens"] > 0 or cache_read_input_tokens is None + _reports_prompt_side_usage(usage_chunk_dict) or cache_read_input_tokens is None ): cache_read_input_tokens = usage_chunk_dict["cache_read_input_tokens"] if usage_chunk_dict["completion_tokens_details"] is not None: diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index bf37b1be2e4..5d7956059e4 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -10,7 +10,6 @@ import anyio import anyio.lowlevel import httpx import tiktoken -from tokenizers import Tokenizer from typing_extensions import ParamSpec, TypeVar import litellm @@ -30,8 +29,10 @@ from litellm.constants import ( TOKEN_COUNTER_MAX_EXACT_CHARS, ) from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, HuggingFaceTokenizer, OpenAIEncoding from litellm.litellm_core_utils.url_utils import safe_get from litellm.llms.custom_httpx.http_handler import _get_httpx_client +from litellm.rust_bridge.tokenizer import get_encoding from litellm.types.llms.anthropic import ( AnthropicContentParamSource, AnthropicContentParamSourceFileId, @@ -622,9 +623,11 @@ def _get_exact_count_function( if model is not None or custom_tokenizer is not None: tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model) if tokenizer_json["type"] == "huggingface_tokenizer": - tokenizer: Final[Tokenizer] = tokenizer_json["tokenizer"] + tokenizer: Final[HuggingFace] = tokenizer_json["tokenizer"] def count_tokens(text: str) -> int: + if isinstance(tokenizer, HuggingFaceTokenizer): + return tokenizer.count(text) return len(tokenizer.encode_batch_fast([text])[0]) return count_tokens @@ -632,31 +635,43 @@ def _get_exact_count_function( encoding: Final = openai_tokenizer_encoding(model) def encode_length(text: str) -> int: - return len(encoding.encode(text, disallowed_special=())) + return _encoding_count(encoding, text) return _get_tiktoken_count_function(encode_length) else: raise ValueError("Unsupported tokenizer type") else: + default_encoding: Final = _get_default_encoding() def encode_length(text: str) -> int: - return len(_get_default_encoding().encode(text, disallowed_special=())) + return _encoding_count(default_encoding, text) return _get_tiktoken_count_function(encode_length) -def openai_tokenizer_encoding(model: str) -> tiktoken.Encoding: - """The tiktoken encoding `token_counter` uses for a model on the `openai_tokenizer` path.""" +def _encoding_count(encoding: Encoding, text: str) -> int: + if isinstance(encoding, OpenAIEncoding): + return encoding.count(text) + return len(encoding.encode(text, disallowed_special=())) + + +def openai_tokenizer_encoding(model: str) -> Encoding: + """The encoding `token_counter` uses for a model on the `openai_tokenizer` path.""" + return get_encoding(openai_tokenizer_encoding_name(model)) + + +def openai_tokenizer_encoding_name(model: str) -> str: + """The tiktoken encoding name for `model`, without loading the encoding.""" from litellm.utils import print_verbose model_to_use: Final = _fix_model_name(model) if "gpt-4o" in model_to_use: - return tiktoken.get_encoding("o200k_base") + return "o200k_base" try: - return tiktoken.encoding_for_model(model_to_use) + return tiktoken.encoding_name_for_model(model_to_use) except KeyError: print_verbose("Warning: model not found. Using cl100k_base encoding.") - return tiktoken.get_encoding("cl100k_base") + return "cl100k_base" def uses_legacy_message_accounting(model: str) -> bool: diff --git a/litellm/litellm_core_utils/tokenizer.py b/litellm/litellm_core_utils/tokenizer.py new file mode 100644 index 00000000000..aea187fa08e --- /dev/null +++ b/litellm/litellm_core_utils/tokenizer.py @@ -0,0 +1,402 @@ +"""Python faces of the Rust text codecs. + +``OpenAIEncoding`` mirrors ``tiktoken.Encoding`` and ``HuggingFaceTokenizer`` mirrors +``tokenizers.Tokenizer``, so a caller holding ``litellm.encoding`` or the object returned by +``litellm.create_tokenizer`` sees the same read-only surface whichever backend the Rust catalog +selected. Both wrappers are immutable: ``tokenizers`` mutators (``enable_padding``, +``enable_truncation``, ``add_tokens``) stay on the Python tokenizer. +""" + +from __future__ import annotations + +from collections.abc import Callable, Collection, Mapping, Sequence, Set +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, runtime_checkable + +import tiktoken +from tokenizers import AddedToken +from tokenizers import Tokenizer as PythonHuggingFaceTokenizer + +if TYPE_CHECKING: + import numpy as np + import numpy.typing as npt + + from litellm.rust_bridge._native import HuggingFaceEncoding + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + +SpecialTokens: TypeAlias = Literal["all"] | Collection[str] +AllowedSpecial: TypeAlias = Literal["all"] | Set[str] +HuggingFaceInput: TypeAlias = str | list[str] | tuple[str, ...] +HuggingFaceBatchInput: TypeAlias = HuggingFaceInput | tuple[HuggingFaceInput, HuggingFaceInput] | list[HuggingFaceInput] + + +@dataclass(frozen=True, slots=True) +class OpenAIEncoding: + """``tiktoken.Encoding`` over the Rust tiktoken codec.""" + + _native: NativeTokenizer + _special_tokens: Mapping[str, int] + + @staticmethod + def wrap(native: NativeTokenizer) -> OpenAIEncoding: + return OpenAIEncoding(native, MappingProxyType(native.special_tokens())) + + @staticmethod + def from_tiktoken(encoding: str) -> OpenAIEncoding: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + + return OpenAIEncoding.wrap(NativeTokenizer.from_tiktoken(encoding)) + + def __repr__(self) -> str: + return f"" + + @property + def name(self) -> str: + return self._native.name + + @property + def max_token_value(self) -> int: + return self._native.max_token_value() + + @property + def n_vocab(self) -> int: + """For backwards compatibility. Prefer to use `enc.max_token_value + 1`.""" + return self.max_token_value + 1 + + @property + def eot_token(self) -> int: + return self._special_tokens["<|endoftext|>"] + + @property + def special_tokens_set(self) -> set[str]: # mutable-ok: [LIT001, LIT002] SDK return type + return set(self._special_tokens) + + def is_special_token(self, token: int) -> bool: + return self._native.is_special_token(token) + + # ---- encoding ------------------------------------------------------------------------- + + def encode_ordinary(self, text: str) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.encode(text) + + def encode( + self, + text: str, + *, + allowed_special: AllowedSpecial = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type + allowed: Final = self._allowed(text, allowed_special, disallowed_special) + if not allowed: + return self.encode_ordinary(text) + return self._native.encode_special(text, tuple(allowed)) + + def encode_to_numpy( + self, + text: str, + *, + allowed_special: AllowedSpecial = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> npt.NDArray[np.uint32]: + import numpy + + return numpy.asarray( + self.encode(text, allowed_special=allowed_special, disallowed_special=disallowed_special), + dtype=numpy.uint32, + ) + + def encode_ordinary_batch( + self, text: Sequence[str], *, num_threads: int = 8 + ) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(self.encode_ordinary, text) + ) + + def encode_batch( + self, + text: Sequence[str], + *, + num_threads: int = 8, + allowed_special: AllowedSpecial = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type + encode: Final = partial(self.encode, allowed_special=allowed_special, disallowed_special=disallowed_special) + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(encode, text) + ) + + def encode_with_unstable( + self, + text: str, + *, + allowed_special: AllowedSpecial = frozenset(), + disallowed_special: SpecialTokens = "all", + ) -> tuple[list[int], list[list[int]]]: # mutable-ok: [LIT001, LIT002] SDK return type + """The stable tokens of `text` and every completion its unstable tail could become. + + Completions come back sorted; tiktoken returns them in hash order.""" + allowed: Final = self._allowed(text, allowed_special, disallowed_special) + return self._native.encode_with_unstable(text, tuple(allowed)) + + def encode_single_token(self, text_or_bytes: str | bytes) -> int: + """The token of one whole piece, special tokens included. Raises `KeyError` otherwise.""" + piece: Final = text_or_bytes.encode("utf-8") if isinstance(text_or_bytes, str) else text_or_bytes + return self._native.encode_single_token(piece) + + def count(self, text: str, fast: bool = False) -> int: + """Count ordinary text; `fast` accelerates supported encodings and otherwise counts normally.""" + return self._native.count(text, fast) + + # ---- decoding ------------------------------------------------------------------------- + + def decode_bytes(self, tokens: Sequence[int]) -> bytes: + return self._native.decode_bytes(tokens) + + def decode(self, tokens: Sequence[int], errors: str = "replace") -> str: + return self.decode_bytes(tokens).decode("utf-8", errors=errors) + + def decode_single_token_bytes(self, token: int) -> bytes: + return self.decode_bytes((token,)) + + def decode_tokens_bytes(self, tokens: Sequence[int]) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type + return [ # mutable-ok: [LIT002] SDK returns a list + self.decode_single_token_bytes(token) for token in tokens + ] + + def decode_with_offsets( + self, tokens: Sequence[int] + ) -> tuple[str, list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type + """The decoded text and, per token, the index of the first character holding its bytes. + + Like tiktoken, raises `UnicodeDecodeError` when the tokens do not decode to valid UTF-8.""" + token_bytes: Final = self.decode_tokens_bytes(tokens) + text_len = 0 + offsets: Final[list[int]] = [] # mutable-ok: [LIT001] local accumulator + for token in token_bytes: + offsets.append(max(0, text_len - (0x80 <= token[0] < 0xC0))) + text_len += sum(1 for c in token if not 0x80 <= c < 0xC0) + return b"".join(token_bytes).decode("utf-8", errors="strict"), offsets + + def decode_batch( + self, batch: Sequence[Sequence[int]], *, errors: str = "replace", num_threads: int = 8 + ) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(partial(self.decode, errors=errors), batch) + ) + + def decode_bytes_batch( + self, batch: Sequence[Sequence[int]], *, num_threads: int = 8 + ) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type + with ThreadPoolExecutor(num_threads) as executor: + return list( # mutable-ok: [LIT002] SDK returns a list + executor.map(self.decode_bytes, batch) + ) + + def token_byte_values(self) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.token_byte_values() + + def __reduce__(self) -> tuple[Callable[[str], OpenAIEncoding], tuple[str]]: + return (OpenAIEncoding.from_tiktoken, (self.name,)) + + # ---- private -------------------------------------------------------------------------- + + def _allowed(self, text: str, allowed_special: AllowedSpecial, disallowed_special: SpecialTokens) -> frozenset[str]: + """tiktoken's special-token policy: which specials `text` may encode, after rejecting + any it must not contain.""" + allowed: Final = frozenset(self._special_tokens) if allowed_special == "all" else frozenset(allowed_special) + disallowed: Final = ( + frozenset(self._special_tokens) - allowed if disallowed_special == "all" else frozenset(disallowed_special) + ) + for token in disallowed: + if token in text: + raise ValueError( + f"Encountered text corresponding to disallowed special token {token!r}.\n" + "If you want this text to be encoded as a special token, " + f"pass it to `allowed_special`, e.g. `allowed_special={{{token!r}, ...}}`.\n" + "If you want this text to be encoded as normal text, disable the check for this token " + f"by passing `disallowed_special=(enc.special_tokens_set - {{{token!r}}})`.\n" + "To disable this check for all special tokens, pass `disallowed_special=()`.\n" + ) + return allowed + + +@dataclass(frozen=True, slots=True) +class HuggingFaceTokenizer: + """The read-only ``tokenizers.Tokenizer`` surface over the Rust Hugging Face codec.""" + + _native: NativeTokenizer + + @staticmethod + def from_str(json: str) -> HuggingFaceTokenizer: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + + return HuggingFaceTokenizer(NativeTokenizer.from_json(json)) + + from_json = from_str + + @staticmethod + def from_buffer(buffer: bytes) -> HuggingFaceTokenizer: + return HuggingFaceTokenizer.from_str(buffer.decode("utf-8")) + + @staticmethod + def from_file(path: str) -> HuggingFaceTokenizer: + return HuggingFaceTokenizer.from_str(Path(path).read_text(encoding="utf-8")) + + @staticmethod + def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> HuggingFaceTokenizer: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + + return HuggingFaceTokenizer(NativeTokenizer.from_pretrained(identifier, revision=revision, token=token)) + + def to_str(self, pretty: bool = False) -> str: + return self._native.to_json(pretty) + + def save(self, path: str, pretty: bool = True) -> None: + Path(path).write_text(self.to_str(pretty), encoding="utf-8") + + @property + def name(self) -> str: + return self._native.name + + # ---- vocabulary ----------------------------------------------------------------------- + + def token_to_id(self, token: str) -> int | None: + return self._native.token_to_id(token) + + def id_to_token(self, id: int) -> str | None: + return self._native.id_to_token(id) + + def get_vocab( + self, with_added_tokens: bool = True + ) -> dict[str, int]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.get_vocab(with_added_tokens) + + def get_vocab_size(self, with_added_tokens: bool = True) -> int: + return self._native.get_vocab_size(with_added_tokens) + + def get_added_tokens_decoder(self) -> dict[int, AddedToken]: # mutable-ok: [LIT001, LIT002] SDK return type + return { # mutable-ok: [LIT002] SDK returns a dict + token_id: AddedToken( + content, single_word=single_word, lstrip=lstrip, rstrip=rstrip, normalized=normalized, special=special + ) + for token_id, ( + content, + single_word, + lstrip, + rstrip, + normalized, + special, + ) in self._native.added_tokens_decoder() + } + + def num_special_tokens_to_add(self, is_pair: bool) -> int: + return self._native.num_special_tokens_to_add(is_pair) + + @property + def padding(self) -> dict[str, object] | None: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.padding() + + @property + def truncation(self) -> dict[str, object] | None: # mutable-ok: [LIT001, LIT002] SDK return type + return self._native.truncation() + + @property + def encode_special_tokens(self) -> bool: + return self._native.encode_special_tokens() + + # ---- encoding and decoding ------------------------------------------------------------ + + def encode( + self, + sequence: HuggingFaceInput, + pair: HuggingFaceInput | None = None, + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> HuggingFaceEncoding: + return self._native.encode_huggingface(sequence, pair, is_pretokenized, add_special_tokens) + + def encode_batch( + self, + input: Sequence[HuggingFaceBatchInput], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=False) + + def encode_batch_fast( + self, + input: Sequence[HuggingFaceBatchInput], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type + return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=True) + + def _encode_batch( + self, input: Sequence[HuggingFaceBatchInput], is_pretokenized: bool, add_special_tokens: bool, fast: bool + ) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type + sequences: Final = tuple(_batch_input(item, is_pretokenized) for item in input) + return self._native.encode_batch_huggingface(sequences, is_pretokenized, add_special_tokens, fast) + + def count(self, text: str, fast: bool = False) -> int: + """Count with this tokenizer's configuration; `fast` uses acceleration where supported.""" + return self._native.count(text, fast) + + def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str: + return self._native.decode(ids, skip_special_tokens=skip_special_tokens) + + def decode_batch( + self, sequences: Sequence[Sequence[int]], skip_special_tokens: bool = True + ) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type + return [ # mutable-ok: [LIT002] SDK returns a list + self.decode(ids, skip_special_tokens=skip_special_tokens) for ids in sequences + ] + + def __reduce__(self) -> tuple[Callable[[str], HuggingFaceTokenizer], tuple[str]]: + return (HuggingFaceTokenizer.from_str, (self.to_str(),)) + + +def _batch_input( + item: HuggingFaceBatchInput, is_pretokenized: bool +) -> tuple[HuggingFaceInput, HuggingFaceInput | None]: + if isinstance(item, str): + return (item, None) + if is_pretokenized and all(isinstance(word, str) for word in item): + return (tuple(word for word in item if isinstance(word, str)), None) + if len(item) != 2: + raise TypeError("batch input must be a sequence or a pair of sequences") + return (item[0], item[1]) + + +Encoding: TypeAlias = tiktoken.Encoding | OpenAIEncoding +HuggingFace: TypeAlias = PythonHuggingFaceTokenizer | HuggingFaceTokenizer +Tokenizer: TypeAlias = Encoding | HuggingFace + + +class _AddedToken(Protocol): + @property + def special(self) -> bool: ... + + +@runtime_checkable +class _AddedTokenDecoder(Protocol): + def get_added_tokens_decoder(self) -> Mapping[int, _AddedToken]: ... + + +def strip_special_tokens(tokenizer: object, tokens: Sequence[int]) -> Sequence[int]: + """Drop the special added tokens before a Python `tokenizers` decode; the Rust codec's + `decode(skip_special_tokens=True)` already does this itself.""" + if isinstance(tokenizer, HuggingFaceTokenizer) or not isinstance(tokenizer, _AddedTokenDecoder): + return tokens + try: + added: Final = tokenizer.get_added_tokens_decoder() + except Exception: # noqa: BLE001 # optional metadata failures historically fall back to decoding + return tokens + special_ids: Final = frozenset(token_id for token_id, token in added.items() if token.special) + return tuple(token for token in tokens if token not in special_ids) diff --git a/litellm/litellm_core_utils/url_utils.py b/litellm/litellm_core_utils/url_utils.py index b94d5a6886d..6c87ef4a3de 100644 --- a/litellm/litellm_core_utils/url_utils.py +++ b/litellm/litellm_core_utils/url_utils.py @@ -336,7 +336,7 @@ def validate_url(url: str) -> tuple[str, str]: raise SSRFError( f"URL targets a blocked address ({resolved_ip}). " "If this is a legitimate internal service, add the host " - "to `user_url_allowed_hosts` in general_settings." + "to `user_url_allowed_hosts` in litellm_settings." ) # For HTTPS with SSL verification enabled, TLS certificate validation diff --git a/litellm/llms/a2a/chat/transformation.py b/litellm/llms/a2a/chat/transformation.py index 77f26b65de0..c5a71daaba5 100644 --- a/litellm/llms/a2a/chat/transformation.py +++ b/litellm/llms/a2a/chat/transformation.py @@ -23,9 +23,8 @@ from ..common_utils import ( from .streaming_iterator import A2AModelResponseIterator if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer _REGISTRY_PARAMS_KEPT_OUT_OF_OPTIONAL_PARAMS: Final = ( @@ -292,7 +291,7 @@ class A2AConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/aiml/image_generation/cost_calculator.py b/litellm/llms/aiml/image_generation/cost_calculator.py index 13427dcafc2..abf4216807a 100644 --- a/litellm/llms/aiml/image_generation/cost_calculator.py +++ b/litellm/llms/aiml/image_generation/cost_calculator.py @@ -1,19 +1,22 @@ from typing import Any, Final import litellm -from litellm.types.utils import ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.types.utils import ImageResponse, ModelInfo def cost_calculator( model: str, image_response: Any, + model_info: ModelInfo | None = None, ) -> float: """ AI/ML flux image generation cost calculator """ - _model_info: Final = litellm.get_model_info( + _model_info: Final = resolve_image_model_info( model=model, custom_llm_provider=litellm.LlmProviders.AIML.value, + model_info=model_info, ) output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 diff --git a/litellm/llms/aiml/image_generation/transformation.py b/litellm/llms/aiml/image_generation/transformation.py index 4f4cd074165..b585d35a2ac 100644 --- a/litellm/llms/aiml/image_generation/transformation.py +++ b/litellm/llms/aiml/image_generation/transformation.py @@ -14,9 +14,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -171,7 +170,7 @@ class AimlImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/aiohttp_openai/chat/transformation.py b/litellm/llms/aiohttp_openai/chat/transformation.py index 530896bf9b0..a06c670e3f1 100644 --- a/litellm/llms/aiohttp_openai/chat/transformation.py +++ b/litellm/llms/aiohttp_openai/chat/transformation.py @@ -16,9 +16,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -68,7 +67,7 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/amazon_nova/chat/transformation.py b/litellm/llms/amazon_nova/chat/transformation.py index 7551fb28c21..a93fbf1e933 100644 --- a/litellm/llms/amazon_nova/chat/transformation.py +++ b/litellm/llms/amazon_nova/chat/transformation.py @@ -17,7 +17,7 @@ from litellm.types.utils import ModelResponse from ...openai_like.chat.transformation import OpenAILikeChatConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AmazonNovaChatConfig(OpenAILikeChatConfig): @@ -86,7 +86,7 @@ class AmazonNovaChatConfig(OpenAILikeChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/anthropic/batches/transformation.py b/litellm/llms/anthropic/batches/transformation.py index 4f4d39f09b0..7dee7513538 100644 --- a/litellm/llms/anthropic/batches/transformation.py +++ b/litellm/llms/anthropic/batches/transformation.py @@ -14,9 +14,8 @@ from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest from litellm.types.utils import LiteLLMBatch, LlmProviders, ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -290,7 +289,7 @@ class AnthropicBatchesConfig(BaseBatchesConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 545e920156e..b7c2ce3c568 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -96,13 +96,13 @@ from ..common_utils import ( AnthropicModelInfo, eager_input_streaming_flag, process_anthropic_headers, + requires_native_compaction_beta, strip_advisor_blocks_from_messages, ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -1771,7 +1771,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return tools - def _ensure_beta_header(self, headers: dict, beta_value: str) -> None: + def _ensure_beta_header(self, headers: dict[str, str], beta_value: str) -> None: """ Ensure a beta header value is present in the anthropic-beta header. Merges with existing values instead of overriding them. @@ -1780,13 +1780,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): headers: Dictionary of headers to update beta_value: The beta header value to add """ - existing_beta: Final = headers.get("anthropic-beta") - if existing_beta is None: - headers["anthropic-beta"] = beta_value - return - existing_values: Final = [beta.strip() for beta in existing_beta.split(",")] - if beta_value not in existing_values: - headers["anthropic-beta"] = f"{existing_beta}, {beta_value}" + existing_values: Final = tuple( + beta.strip() + for key, value in headers.items() + if key.lower() == "anthropic-beta" + for beta in value.split(",") + if beta.strip() + ) + for key in tuple(headers): + if key.lower() == "anthropic-beta": + headers.pop(key) + headers["anthropic-beta"] = ", ".join(dict.fromkeys((*existing_values, beta_value))) def _ensure_context_management_beta_header(self, headers: dict, context_management: object) -> None: """ @@ -1824,7 +1828,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, ) - def update_headers_with_optional_anthropic_beta(self, headers: dict, optional_params: dict) -> dict: + def update_headers_with_optional_anthropic_beta( + self, headers: dict, optional_params: dict, messages: Sequence[object] = () + ) -> dict: """Update headers with optional anthropic beta.""" # Skip adding beta headers for Vertex requests @@ -1833,6 +1839,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if is_vertex_request: return headers + if requires_native_compaction_beta(self._resolved_provider, optional_params, messages): + self._ensure_beta_header(headers, ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) + _tools: Final = optional_params.get("tools", []) for tool in _tools: if tool.get("type", None) and tool.get("type").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_FETCH.value): @@ -1929,8 +1938,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): custom_llm_provider=self._resolved_provider, ) - headers = self.update_headers_with_optional_anthropic_beta(headers=headers, optional_params=optional_params) - # === Tool-name sanitization (single chokepoint) === # Anthropic enforces ^[a-zA-Z0-9_-]{1,128}$ on every tool name. We # sanitize *here* -- not in map_openai_params -- because: @@ -1977,6 +1984,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): message=f"{e}\nReceived Messages={messages}", ) # don't use verbose_logger.exception, if exception is raised + self.update_headers_with_optional_anthropic_beta( + headers=headers, optional_params=optional_params, messages=anthropic_messages + ) + ## Auto-strip advisor blocks from history if advisor tool is absent. ## Prevents Anthropic 400: advisor_tool_result in history requires advisor tool. _all_tools: Final = optional_params.get("tools") or [] @@ -2688,7 +2699,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index c6015e7884e..0c3c8996789 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -79,6 +79,27 @@ _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") +def requires_native_compaction_beta( + custom_llm_provider: str, + optional_params: Mapping[str, object], + messages: Sequence[object], +) -> bool: + return custom_llm_provider == "anthropic" and ( + optional_params.get("compaction") is not None + or any( + isinstance(block, Mapping) + and block.get("type") == "compaction" + and isinstance(block.get("signature"), str) + and bool(block.get("signature")) + for message in messages + if isinstance(message, Mapping) + for content in (message.get("content"),) + if isinstance(content, (list, tuple)) + for block in content + ) + ) + + def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool: from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.utils import supports_prompt_caching diff --git a/litellm/llms/anthropic/compaction.py b/litellm/llms/anthropic/compaction.py new file mode 100644 index 00000000000..cd09f936248 --- /dev/null +++ b/litellm/llms/anthropic/compaction.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter + +from litellm.llms.compaction import CompactionProtocol +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, AnthropicCompaction + +_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_OBJECTS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +_HEADERS: Final = TypeAdapter(dict[str, str]) +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) +_CONFLICTS: Final = ("context_management", "response_format", "stop", "stop_sequences", "tool_choice") + + +def supports_native_compaction(params: Mapping[str, object]) -> bool: + from litellm.utils import get_model_info + + if params.get("custom_llm_provider") not in (None, "anthropic", "openai"): + return False + model: Final = str(params.get("model", "")).removeprefix("openai/").removeprefix("anthropic/") + try: + return get_model_info(model=model, custom_llm_provider="anthropic").get("supports_anthropic_compaction") is True + except Exception: + return False + + +def compatible_defaults(payload: Mapping[str, object]) -> bool: + return all(payload.get(key) is None for key in _CONFLICTS) + + +def request_kwargs() -> Mapping[str, object]: + operation: Final[AnthropicCompaction] = {"type": "summarize"} + return MappingProxyType( + { + "compaction": operation, + "extra_headers": _HEADERS.validate_python( + MappingProxyType({"anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value}) + ), + } + ) + + +def _native_blocks(protocol: CompactionProtocol, response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + if protocol == "messages": + return ( + _OBJECTS.validate_python(response.get("content", ())) if response.get("stop_reason") == "compaction" else () + ) + choices: Final = _OBJECTS.validate_python(response.get("choices", ())) + choice: Final = choices[0] if len(choices) == 1 else _EMPTY + message: Final = _MAPPING.validate_python(choice.get("message", _EMPTY)) + fields: Final = _MAPPING.validate_python(message.get("provider_specific_fields") or _EMPTY) + return _OBJECTS.validate_python(fields.get("compaction_blocks", ())) + + +def extract_summary(protocol: CompactionProtocol, response: Mapping[str, object]) -> str | None: + blocks: Final = _native_blocks(protocol, response) + block: Final = blocks[0] if len(blocks) == 1 else _EMPTY + content: Final = block.get("content") + return ( + content + if block.get("type") == "compaction" + and isinstance(block.get("signature"), str) + and block.get("signature") + and isinstance(content, str) + and content.strip() + else None + ) diff --git a/litellm/llms/anthropic/completion/transformation.py b/litellm/llms/anthropic/completion/transformation.py index b15b0159bd9..46ab27ab0c7 100644 --- a/litellm/llms/anthropic/completion/transformation.py +++ b/litellm/llms/anthropic/completion/transformation.py @@ -33,7 +33,7 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AnthropicTextError(BaseLLMException): @@ -185,7 +185,7 @@ class AnthropicTextConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 54d10837d74..116f96cf00c 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -27,6 +27,7 @@ from litellm.llms.anthropic.experimental_pass_through.utils import ( from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) +from litellm.types.llms.openai import OpenAIWebSearchOptions from litellm.types.utils import ModelResponse from litellm.utils import get_model_info @@ -383,6 +384,49 @@ class LiteLLMMessagesToCompletionTransformationHandler: updated_reasoning_effort["summary"] = effective_summary completion_kwargs["reasoning_effort"] = updated_reasoning_effort + @staticmethod + def _plain_effort_for_chat_target( + completion_kwargs: _CompletionKwargs, + *, + thinking: Mapping[str, object] | None, + ) -> str | None: + reasoning_effort: Final = completion_kwargs.get("reasoning_effort") + if not thinking or not isinstance(reasoning_effort, dict) or "summary" not in reasoning_effort: + return None + effort: Final = reasoning_effort.get("effort") + model: Final = completion_kwargs.get("model") + if not isinstance(effort, str) or not isinstance(model, str) or not model: + return None + custom_llm_provider: Final = completion_kwargs.get("custom_llm_provider") + api_base: Final = completion_kwargs.get("api_base") + api_key: Final = completion_kwargs.get("api_key") + try: + local_model, resolved_provider, _, resolved_api_base = litellm.utils.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + api_base=api_base if isinstance(api_base, str) else None, + api_key=api_key if isinstance(api_key, str) else None, + ) + except Exception: + return None + if resolved_provider == "litellm_proxy": + return None + from litellm.main import responses_api_bridge_check + + web_search_options: Final = completion_kwargs.get("web_search_options") + tools: Final = completion_kwargs.get("tools") + model_info, _ = responses_api_bridge_check( + model=local_model, + custom_llm_provider=resolved_provider, + web_search_options=( + cast(OpenAIWebSearchOptions, web_search_options) if isinstance(web_search_options, dict) else None + ), + tools=cast("list[dict[str, object]]", tools) if isinstance(tools, list) else None, + reasoning_effort=reasoning_effort, + api_base=resolved_api_base, + ) + return None if model_info.get("mode") == "responses" else effort + @staticmethod def _normalize_reasoning_effort( completion_kwargs: _CompletionKwargs, @@ -547,6 +591,13 @@ class LiteLLMMessagesToCompletionTransformationHandler: thinking=thinking, ) + plain_effort: Final = LiteLLMMessagesToCompletionTransformationHandler._plain_effort_for_chat_target( + completion_kwargs, + thinking=thinking, + ) + if plain_effort is not None: + completion_kwargs["reasoning_effort"] = plain_effort + return completion_kwargs, tool_name_mapping @staticmethod diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 1a85cf80bff..85431a5a637 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -2,8 +2,11 @@ import copy import hashlib import json from collections.abc import AsyncIterator, Iterator, Mapping, Sequence +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast +from pydantic import JsonValue, TypeAdapter + import litellm from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, @@ -16,6 +19,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64 TOOL_NAME_HASH_LENGTH: Final = 8 TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55 PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"}) +_COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue]) def _optional_attr(source: object, name: str) -> object: @@ -36,6 +40,20 @@ def _thought_signature(provider_specific_fields: object) -> str | None: return signature if isinstance(signature, str) else None +def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]: + fields: Final = _as_string_mapping(provider_specific_fields) + raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None + return ( + tuple( + block + for raw_block in raw_blocks + if (block := _as_string_mapping(raw_block)) is not None and block.get("type") == "compaction" + ) + if isinstance(raw_blocks, (list, tuple)) + else () + ) + + _ANTHROPIC_TOOL_SCHEMA_KEYS: Final = frozenset( {"name", "type", "input_schema", "description", "cache_control", "strict"} ) @@ -1330,7 +1348,11 @@ class LiteLLMAnthropicMessagesAdapter: tool_name_mapping: dict[str, str] | None = None, ) -> list[dict[str, Any]]: new_content: Final[list[dict[str, Any]]] = [] - for choice in choices: + for choice, compaction_blocks in ( + (choice, _compaction_blocks(_optional_attr(choice.message, "provider_specific_fields"))) + for choice in choices + ): + new_content.extend(_COMPACTION_BLOCK.validate_python(block) for block in compaction_blocks) # Handle thinking blocks first if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks: for thinking_block in choice.message.thinking_blocks: @@ -1365,7 +1387,7 @@ class LiteLLMAnthropicMessagesAdapter: ) # Handle text content - if choice.message.content is not None: + if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks): new_content.append( AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump() ) @@ -1545,21 +1567,35 @@ class LiteLLMAnthropicMessagesAdapter: openai_finish_reason=openai_finish_reason ) anthropic_finish_reason: Final = ( - "refusal" + "compaction" + if len(anthropic_content) == 1 and anthropic_content[0].get("type") == "compaction" + else "refusal" if refusal_text is not None and translated_finish_reason != "max_tokens" else translated_finish_reason ) # extract usage usage: Final[Usage] = getattr(response, "usage") - anthropic_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage) - - if polyfill_result is not None and polyfill_result.iterations_usage is not None: - message_iteration: Final[UsageIteration] = { - "type": "message", - "input_tokens": anthropic_usage["input_tokens"], - "output_tokens": usage.completion_tokens or 0, - } - anthropic_usage["iterations"] = list(polyfill_result.iterations_usage) + [message_iteration] + message_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage) + polyfill_iterations: Final = polyfill_result.iterations_usage if polyfill_result is not None else None + anthropic_usage: Final[AnthropicUsage] = ( + TypeAdapter(AnthropicUsage).validate_python( + MappingProxyType( + { + **message_usage, + "iterations": ( + *polyfill_iterations, + UsageIteration( + type="message", + input_tokens=message_usage.get("input_tokens", 0), + output_tokens=usage.completion_tokens or 0, + ), + ), + } + ) + ) + if polyfill_iterations is not None + else message_usage + ) translated_obj: Final = AnthropicMessagesResponse( id=response.id, diff --git a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py index ebd0342b10c..f23f2602ba8 100644 --- a/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py +++ b/litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py @@ -208,9 +208,12 @@ async def _check_summary_model_access( (``ProxyException`` from ``_can_object_call_model`` / ``can_*_model``). Unexpected errors during an access check fail closed but are logged separately so operators can distinguish them from a real access-denied - response. DB-lookup failures (object missing from cache or DB) skip the - corresponding scope — matching ``common_checks``, which only enforces a - scope when its backing object can be loaded. + response. User and project lookup failures (object missing from cache or + DB) skip the corresponding scope — matching ``common_checks``, which only + enforces a scope when its backing object can be loaded. A failed team + membership read (a database outage) fails closed instead, since a member + whose limits cannot be read must not have the summary model invoked with + those limits dropped. """ if user_api_key_auth is None: return True @@ -346,13 +349,12 @@ async def _check_summary_model_access( proxy_logging_obj=proxy_logging_obj, ) except Exception as e: - verbose_logger.debug( - "compact_20260112: team membership lookup failed for " - "summary_model=%s access check; skipping member-level scope: %s", + verbose_logger.warning( + "compact_20260112: team membership lookup failed for summary_model=%s access check; denying access: %s", summary_model, e, ) - team_membership = None + return False member_allowed_models: Final = ( team_membership.litellm_budget_table.allowed_models if team_membership is not None and team_membership.litellm_budget_table is not None diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index d87cb0a64f5..ac4240690c1 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -580,7 +580,9 @@ def anthropic_messages_handler( ) if anthropic_messages_provider_config is None: # Route to Responses API for OpenAI / Azure, chat/completions for everything else. - if _should_route_to_responses_api(custom_llm_provider, original_model, model): + if kwargs.get("compaction") is None and _should_route_to_responses_api( + custom_llm_provider, original_model, model + ): return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( max_tokens=max_tokens, messages=messages, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index eed30c2698c..a83e23d83d5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -24,6 +24,7 @@ from ...common_utils import ( AnthropicError, AnthropicModelInfo, optionally_handle_anthropic_oauth, + requires_native_compaction_beta, strip_advisor_blocks_from_messages, strip_encrypted_reasoning_blocks_from_anthropic_messages, ) @@ -74,6 +75,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): "tool_choice", "thinking", "context_management", + *(("compaction",) if self._resolved_provider == "anthropic" else ()), "output_format", "inference_geo", "speed", @@ -637,6 +639,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): ) beta_values.update(existing_beta) + if requires_native_compaction_beta(custom_llm_provider, optional_params, messages): + beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) + # Check for context management context_management_param: Final = optional_params.get("context_management") if context_management_param is not None: diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 424422612db..355714c0daf 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -29,9 +29,8 @@ from ...base_llm.chat.transformation import BaseConfig from ..common_utils import AzureOpenAIError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -304,7 +303,7 @@ class AzureOpenAIConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py index 1b5a4083ebe..c40cefecdd0 100644 --- a/litellm/llms/azure/passthrough/transformation.py +++ b/litellm/llms/azure/passthrough/transformation.py @@ -143,7 +143,7 @@ class AzurePassthroughConfig(BasePassthroughConfig): ) -> dict: return BaseAzureLLM._base_validate_azure_environment( headers=headers, - litellm_params=GenericLiteLLMParams(**{**litellm_params, "api_key": api_key}), + litellm_params=GenericLiteLLMParams.model_validate({**litellm_params, "api_key": api_key}), ) @staticmethod diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 146915dd6fd..df74975ad0f 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -8,11 +8,15 @@ from collections.abc import Mapping from types import MappingProxyType from typing import Any, Final, Protocol, cast -from litellm._logging import _redact_string, verbose_proxy_logger +from litellm._logging import verbose_proxy_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.types.realtime import RealtimeQueryParams from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from ....litellm_core_utils.realtime_errors import ( + close_after_upstream_handshake_refusal, + realtime_error_event, +) from ....litellm_core_utils.realtime_streaming import ( RealTimeStreaming, ScopedWebSocket, @@ -49,7 +53,9 @@ def azure_realtime_protocol_for_client( class _ProxyClientWebSocket(Protocol): - """Client-facing websocket handle: this path only closes it after a failed handshake.""" + """Client-facing websocket handle: this path only writes to it after a failed handshake.""" + + async def send_text(self, data: str) -> None: ... async def close(self, code: int = ..., reason: str | None = ...) -> None: ... @@ -181,7 +187,16 @@ class AzureOpenAIRealtime(AzureChatCompletion): ) await realtime_streaming.bidirectional_forward() - except websockets.exceptions.InvalidStatusCode as e: - await websocket.close(code=e.status_code, reason=_redact_string(str(e))) + except websockets.exceptions.InvalidStatus as e: + verbose_proxy_logger.exception("Error in AzureOpenAIRealtime.async_realtime") + await close_after_upstream_handshake_refusal(websocket, e.response.status_code) except Exception: verbose_proxy_logger.exception("Error in AzureOpenAIRealtime.async_realtime") + try: + await websocket.send_text(realtime_error_event("Internal server error", error_type="server_error")) + except Exception: # noqa: BLE001 # best-effort notice: a dead client socket must not skip the close below + pass + try: + await websocket.close(code=1011, reason="Internal server error") + except Exception: # noqa: BLE001 # the lower layer may have closed the socket already; closing twice is not an error + pass diff --git a/litellm/llms/azure_ai/agents/transformation.py b/litellm/llms/azure_ai/agents/transformation.py index 60ce81a23c7..baba3149963 100644 --- a/litellm/llms/azure_ai/agents/transformation.py +++ b/litellm/llms/azure_ai/agents/transformation.py @@ -34,9 +34,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler LiteLLMLoggingObj = _LiteLLMLoggingObj @@ -297,7 +296,7 @@ class AzureAIAgentsConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure_ai/azure_model_router/transformation.py b/litellm/llms/azure_ai/azure_model_router/transformation.py index 9e35e396e15..0e4c8ca0d15 100644 --- a/litellm/llms/azure_ai/azure_model_router/transformation.py +++ b/litellm/llms/azure_ai/azure_model_router/transformation.py @@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AzureModelRouterConfig(AzureAIStudioConfig): @@ -59,7 +59,7 @@ class AzureModelRouterConfig(AzureAIStudioConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 00e1c1e25ba..779a86629e2 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -30,7 +30,7 @@ from litellm.types.utils import ModelResponse, ProviderField from litellm.utils import _add_path_to_api_base, supports_tool_choice if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AzureFoundryErrorStrings(str, enum.Enum): @@ -305,7 +305,7 @@ class AzureAIStudioConfig(OpenAIConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/azure_ai/image_generation/cost_calculator.py b/litellm/llms/azure_ai/image_generation/cost_calculator.py index 35d0f4fb6c3..08b732197f3 100644 --- a/litellm/llms/azure_ai/image_generation/cost_calculator.py +++ b/litellm/llms/azure_ai/image_generation/cost_calculator.py @@ -3,9 +3,22 @@ from typing import Any, Final import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import ( + _get_cost_per_unit, calculate_image_response_cost_from_usage, + resolve_image_model_info, ) -from litellm.types.utils import ImageResponse +from litellm.types.utils import ImageResponse, ModelInfo + + +def _input_cost_per_pixel(resolved: ModelInfo) -> float: + deployment_price: Final = _get_cost_per_unit(resolved, "input_cost_per_pixel", default_value=None) + if deployment_price is not None: + return deployment_price + model_cost_key: Final = resolved.get("key") + shared_entry: Final = litellm.model_cost.get(model_cost_key) if model_cost_key is not None else None + if shared_entry is None: + return 0.0 + return shared_entry.get("input_cost_per_pixel") or 0.0 def cost_calculator( @@ -14,13 +27,15 @@ def cost_calculator( size: str | None = None, n: int | None = None, optional_params: Mapping[str, object] | None = None, + model_info: ModelInfo | None = None, ) -> float: """ Azure AI image generation cost calculator """ - _model_info: Final = litellm.get_model_info( + _model_info: Final = resolve_image_model_info( model=model, custom_llm_provider=litellm.LlmProviders.AZURE_AI.value, + model_info=model_info, ) if isinstance(image_response, ImageResponse): @@ -28,6 +43,7 @@ def cost_calculator( model=model, image_response=image_response, custom_llm_provider=litellm.LlmProviders.AZURE_AI.value, + model_info=_model_info, ) if token_based_cost is not None: return token_based_cost @@ -37,9 +53,7 @@ def cost_calculator( if output_cost_per_image: return output_cost_per_image * num_images - model_cost: Final = litellm.model_cost[_model_info["key"]] - input_cost_per_pixel: Final[float] = model_cost.get("input_cost_per_pixel") or 0.0 - if input_cost_per_pixel: + if _input_cost_per_pixel(_model_info): from litellm.cost_calculator import default_image_cost_calculator width: Final = optional_params.get("width") if optional_params else None @@ -50,10 +64,11 @@ def cost_calculator( else size or image_response.size ) return default_image_cost_calculator( - model=_model_info["key"], + model=_model_info.get("key", model), custom_llm_provider=litellm.LlmProviders.AZURE_AI.value, size=pixel_size, n=num_images, + model_info=model_info, ) return 0.0 diff --git a/litellm/llms/azure_ai/image_generation/mai_transformation.py b/litellm/llms/azure_ai/image_generation/mai_transformation.py index 67b1a8bcab3..18b4b6f456a 100644 --- a/litellm/llms/azure_ai/image_generation/mai_transformation.py +++ b/litellm/llms/azure_ai/image_generation/mai_transformation.py @@ -12,9 +12,10 @@ from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: - import tiktoken from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): """Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5).""" @@ -245,7 +246,7 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index 2296909cfe1..e4bf148abf3 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -12,9 +12,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -121,7 +120,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/bridges/completion_transformation.py b/litellm/llms/base_llm/bridges/completion_transformation.py index 87b55152d09..a5c03705088 100644 --- a/litellm/llms/base_llm/bridges/completion_transformation.py +++ b/litellm/llms/base_llm/bridges/completion_transformation.py @@ -7,10 +7,10 @@ from collections.abc import AsyncIterator, Iterator from typing import TYPE_CHECKING, Union if TYPE_CHECKING: - import tiktoken from pydantic import BaseModel from litellm import LiteLLMLoggingObj, ModelResponse + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.types.llms.openai import AllMessageValues @@ -39,7 +39,7 @@ class CompletionTransformationBridge(ABC): messages: list["AllMessageValues"], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> "ModelResponse": diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 7bfc87a30d6..7decf1b4186 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -21,9 +21,8 @@ from litellm.types.llms.openai import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.types.utils import ModelResponse from ..base_utils import ( @@ -344,7 +343,7 @@ class BaseConfig(ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> "ModelResponse": diff --git a/litellm/llms/base_llm/completion/transformation.py b/litellm/llms/base_llm/completion/transformation.py index fb472dfa63b..b8ebbfed12f 100644 --- a/litellm/llms/base_llm/completion/transformation.py +++ b/litellm/llms/base_llm/completion/transformation.py @@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -68,7 +67,7 @@ class BaseTextCompletionConfig(BaseConfig, ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/embedding/transformation.py b/litellm/llms/base_llm/embedding/transformation.py index da87dcc7f98..46ac3ccf433 100644 --- a/litellm/llms/base_llm/embedding/transformation.py +++ b/litellm/llms/base_llm/embedding/transformation.py @@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues from litellm.types.utils import EmbeddingResponse, ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -80,7 +79,7 @@ class BaseEmbeddingConfig(BaseConfig, ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/files/transformation.py b/litellm/llms/base_llm/files/transformation.py index 254995c028f..5ecf033fb2c 100644 --- a/litellm/llms/base_llm/files/transformation.py +++ b/litellm/llms/base_llm/files/transformation.py @@ -21,9 +21,8 @@ from litellm.types.utils import LlmProviders, ModelResponse from ..chat.transformation import BaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.router import Router as _Router from litellm.types.llms.openai import HttpxBinaryResponseContent @@ -231,7 +230,7 @@ class BaseFilesConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/image_generation/transformation.py b/litellm/llms/base_llm/image_generation/transformation.py index 4616441133e..7ac440c6f48 100644 --- a/litellm/llms/base_llm/image_generation/transformation.py +++ b/litellm/llms/base_llm/image_generation/transformation.py @@ -11,9 +11,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -93,7 +92,7 @@ class BaseImageGenerationConfig(ABC): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/base_llm/image_variations/transformation.py b/litellm/llms/base_llm/image_variations/transformation.py index d3e02139e0e..15a4e0f243c 100644 --- a/litellm/llms/base_llm/image_variations/transformation.py +++ b/litellm/llms/base_llm/image_variations/transformation.py @@ -17,9 +17,8 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -82,7 +81,7 @@ class BaseImageVariationConfig(BaseConfig, ABC): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: pass @@ -98,7 +97,7 @@ class BaseImageVariationConfig(BaseConfig, ABC): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: pass @@ -125,7 +124,7 @@ class BaseImageVariationConfig(BaseConfig, ABC): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/base_llm/videos/transformation.py b/litellm/llms/base_llm/videos/transformation.py index 4f94cec0973..a4964de46b8 100644 --- a/litellm/llms/base_llm/videos/transformation.py +++ b/litellm/llms/base_llm/videos/transformation.py @@ -12,6 +12,7 @@ from litellm.types.videos.main import VideoCreateOptionalRequestParams if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.videos.main import CharacterObject as _CharacterObject from litellm.types.videos.main import VideoObject as _VideoObject @@ -269,6 +270,7 @@ class BaseVideoConfig(ABC): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, custom_llm_provider: str | None = None, + client: "HTTPHandler | None" = None, ) -> VideoObject: pass @@ -277,6 +279,7 @@ class BaseVideoConfig(ABC): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, custom_llm_provider: str | None = None, + client: "AsyncHTTPHandler | None" = None, ) -> VideoObject: """Async transform video status retrieve response.""" return self.transform_video_status_retrieve_response( diff --git a/litellm/llms/bedrock/audio_transcription/__init__.py b/litellm/llms/bedrock/audio_transcription/__init__.py index 8f35b8eac7a..172316027a8 100644 --- a/litellm/llms/bedrock/audio_transcription/__init__.py +++ b/litellm/llms/bedrock/audio_transcription/__init__.py @@ -5,7 +5,7 @@ import httpx from litellm.litellm_core_utils.audio_utils.utils import process_audio_file from litellm.rust_bridge import runtime -from litellm.rust_bridge.catalog import Context, Route +from litellm.rust_bridge.catalog import Route, RouteContext from litellm.rust_bridge.timeouts import timeout_to_seconds from litellm.rust_bridge.transcription.native import ( NATIVE_ATRANSCRIPTION, @@ -74,7 +74,7 @@ class BedrockAudioTranscriptionRustDispatch: ) return runtime.run( - Context(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model), + RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model), binding=NATIVE_TRANSCRIPTION, native=native, python=_no_python_implementation, @@ -107,7 +107,7 @@ class BedrockAudioTranscriptionRustDispatch: ) return await runtime.arun( - Context(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model), + RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model), binding=NATIVE_ATRANSCRIPTION, native=native, python=_no_async_python_implementation, diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index dd62cdb424a..badb76d00c7 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -524,6 +524,17 @@ class BaseAWSLLM(SignsRequestsWithAWS): aws_session_tags=_canonical_aws_session_tags(auth_params.aws_session_tags), ) + def resolve_s3_credentials(self, params: Mapping[str, object], aws_region_name: str | None) -> Credentials: + """S3 signing identity: the s3_* static pair as-is when both are set, otherwise the resolved aws_* params.""" + from botocore.credentials import Credentials + + from litellm.llms.bedrock.common_utils import s3_static_key_pair + + s3_pair: Final = s3_static_key_pair(params) + if s3_pair is None: + return self.resolve_credentials(AwsAuthParams.model_validate(params), aws_region_name) + return Credentials(access_key=s3_pair[0], secret_key=s3_pair[1]) + def _get_aws_region_from_model_arn(self, model: str | None) -> str | None: try: # First check if the string contains the expected prefix diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index e1a9a807abc..29133bcfaf9 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -40,9 +40,8 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler LiteLLMLoggingObj = _LiteLLMLoggingObj @@ -990,7 +989,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 4f9b1f56a5b..21830eb0d8e 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -99,7 +99,7 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer # Computer use tool prefixes supported by Bedrock BEDROCK_COMPUTER_USE_TOOLS: Final = [ @@ -1920,7 +1920,7 @@ class AmazonConverseConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_agent/transformation.py b/litellm/llms/bedrock/chat/invoke_agent/transformation.py index d489e47c3b5..6877af74494 100644 --- a/litellm/llms/bedrock/chat/invoke_agent/transformation.py +++ b/litellm/llms/bedrock/chat/invoke_agent/transformation.py @@ -37,9 +37,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -438,7 +437,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py index 5a3f4f17b8b..5699f94d084 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py @@ -25,7 +25,7 @@ from litellm.types.utils import ( from .amazon_llama_transformation import AmazonLlamaConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AmazonDeepSeekR1Config(AmazonLlamaConfig): @@ -39,7 +39,7 @@ class AmazonDeepSeekR1Config(AmazonLlamaConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py index 5d39b68d9d5..f8b730b6cd0 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_moonshot_transformation.py @@ -21,9 +21,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.types.utils import ModelResponse LiteLLMLoggingObj = _LiteLLMLoggingObj @@ -198,7 +197,7 @@ class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> "ModelResponse": diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py index bc97551d57a..8d1ff1d2bd9 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py @@ -28,7 +28,7 @@ from ..converse_transformation import AmazonConverseConfig from .base_invoke_transformation import AmazonInvokeConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer _CachePointCarrier = TypeVar("_CachePointCarrier", SystemContentBlock, ContentBlock) _INJECTION_POINTS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) @@ -128,7 +128,7 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py index c78375c37bb..67364ccfda0 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py @@ -21,7 +21,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AmazonQwen2Config(AmazonQwen3Config): @@ -44,7 +44,7 @@ class AmazonQwen2Config(AmazonQwen3Config): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py index e251fb15725..2e19dbf77af 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py @@ -19,7 +19,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig): @@ -170,7 +170,7 @@ class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py index 39cded4ed64..fe287111fdd 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py @@ -25,9 +25,8 @@ from litellm.types.utils import ModelResponse, Usage from litellm.utils import get_base64_str if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -190,7 +189,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 1326dc22ca0..a8b94fb5703 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -34,9 +34,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -359,7 +358,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 90a2692f68a..dcc5e249d8a 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -34,9 +34,8 @@ from litellm.types.utils import ModelResponse, Usage from litellm.utils import CustomStreamWrapper if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -288,7 +287,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/bedrock/claude_platform/common_utils.py b/litellm/llms/bedrock/claude_platform/common_utils.py index fb7f2185ec5..e25e58056f5 100644 --- a/litellm/llms/bedrock/claude_platform/common_utils.py +++ b/litellm/llms/bedrock/claude_platform/common_utils.py @@ -1,15 +1,76 @@ +from collections.abc import Mapping from typing import Final import httpx import litellm +from litellm._logging import verbose_logger from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import BedrockError from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams CLAUDE_PLATFORM_SERVICE_NAME: Final = "aws-external-anthropic" CLAUDE_PLATFORM_BEDROCK_ROUTE: Final = "claude_platform/" +CLAUDE_PLATFORM_UNSUPPORTED_PARAMS_OVERRIDE_KEY: Final = "claude_platform_unsupported_params" +CLAUDE_PLATFORM_ON_AWS_NON_REQUEST_PARAMS: Final = frozenset( + { + "workspace_id", + "aws_workspace_id", + "anthropic_workspace_id", + "anthropic-workspace-id", + CLAUDE_PLATFORM_UNSUPPORTED_PARAMS_OVERRIDE_KEY, + } +) +CLAUDE_PLATFORM_ON_AWS_UNSUPPORTED_REQUEST_PARAMS: Final = frozenset({"context_management"}) + + +def filter_claude_platform_request_body( + params: Mapping[str, object], + unsupported_override: frozenset[str] | None = None, + log_dropped: bool = True, +) -> dict[str, object]: + unsupported: Final = ( + unsupported_override if unsupported_override is not None else CLAUDE_PLATFORM_ON_AWS_UNSUPPORTED_REQUEST_PARAMS + ) + dropped_unsupported: Final = tuple(k for k in params if k in unsupported) + if dropped_unsupported and log_dropped: + verbose_logger.warning( + "bedrock/claude_platform: dropping unsupported Messages API param(s) %s from the request body; " + "the Claude Platform on AWS endpoint rejects unknown fields. The request will proceed without them.", + dropped_unsupported, + ) + return { + k: v + for k, v in params.items() + if k not in CLAUDE_PLATFORM_ON_AWS_NON_REQUEST_PARAMS and k not in unsupported and not k.startswith("aws_") + } + + +def resolve_unsupported_override( + litellm_params: Mapping[str, object] | GenericLiteLLMParams, + optional_params: Mapping[str, object] | None = None, + log_invalid: bool = True, +) -> frozenset[str] | None: + from_optional: Final = (optional_params or {}).get(CLAUDE_PLATFORM_UNSUPPORTED_PARAMS_OVERRIDE_KEY) + raw: Final = ( + from_optional + if from_optional is not None + else litellm_params.get(CLAUDE_PLATFORM_UNSUPPORTED_PARAMS_OVERRIDE_KEY) + ) + if raw is None: + return None + if isinstance(raw, (list, set, frozenset, tuple)): + return frozenset(str(item) for item in raw) + if log_invalid: + verbose_logger.warning( + "bedrock/claude_platform: ignoring claude_platform_unsupported_params of type %s; " + "expected a list of param names. Using the default unsupported-param set.", + type(raw).__name__, + ) + return None + def strip_claude_platform_route(model: str) -> str: if model.startswith(CLAUDE_PLATFORM_BEDROCK_ROUTE): diff --git a/litellm/llms/bedrock/claude_platform/messages_transformation.py b/litellm/llms/bedrock/claude_platform/messages_transformation.py index 1e3eea075f3..f423d22589b 100644 --- a/litellm/llms/bedrock/claude_platform/messages_transformation.py +++ b/litellm/llms/bedrock/claude_platform/messages_transformation.py @@ -8,7 +8,12 @@ from litellm.llms.anthropic.experimental_pass_through.messages.transformation im from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams -from .common_utils import BedrockClaudePlatformMixin, strip_claude_platform_route +from .common_utils import ( + BedrockClaudePlatformMixin, + filter_claude_platform_request_body, + resolve_unsupported_override, + strip_claude_platform_route, +) class BedrockClaudePlatformMessagesConfig(BedrockClaudePlatformMixin, AnthropicMessagesConfig): @@ -46,26 +51,38 @@ class BedrockClaudePlatformMessagesConfig(BedrockClaudePlatformMixin, AnthropicM if resolved_api_key and "x-api-key" not in headers: headers["x-api-key"] = resolved_api_key - headers = self._update_headers_with_anthropic_beta( - headers=headers, - optional_params=optional_params, - messages=messages, + return ( + self._update_headers_with_anthropic_beta( + headers=headers, + optional_params=filter_claude_platform_request_body( + optional_params, + unsupported_override=resolve_unsupported_override( + litellm_params, optional_params=optional_params, log_invalid=False + ), + log_dropped=False, + ), + messages=messages, + ), + api_base, ) - return headers, api_base - def transform_anthropic_messages_request( self, model: str, - messages: list[dict], - anthropic_messages_optional_request_params: dict, + messages: list[dict[str, object]], + anthropic_messages_optional_request_params: dict[str, object], litellm_params: GenericLiteLLMParams, - headers: dict, - ) -> dict: + headers: dict[str, str], + ) -> dict[str, object]: return super().transform_anthropic_messages_request( model=strip_claude_platform_route(model), messages=messages, - anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + anthropic_messages_optional_request_params=filter_claude_platform_request_body( + anthropic_messages_optional_request_params, + unsupported_override=resolve_unsupported_override( + litellm_params, optional_params=anthropic_messages_optional_request_params + ), + ), litellm_params=litellm_params, headers=headers, ) diff --git a/litellm/llms/bedrock/claude_platform/transformation.py b/litellm/llms/bedrock/claude_platform/transformation.py index a57f309b605..96a6ee4a701 100644 --- a/litellm/llms/bedrock/claude_platform/transformation.py +++ b/litellm/llms/bedrock/claude_platform/transformation.py @@ -5,7 +5,11 @@ from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues -from .common_utils import BedrockClaudePlatformMixin +from .common_utils import ( + BedrockClaudePlatformMixin, + filter_claude_platform_request_body, + resolve_unsupported_override, +) class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): @@ -66,6 +70,25 @@ class BedrockClaudePlatformConfig(BedrockClaudePlatformMixin, AnthropicConfig): anthropic_headers["anthropic-workspace-id"] = workspace_id return {**headers, **anthropic_headers} + def transform_request( + self, + model: str, + messages: list[AllMessageValues], + optional_params: dict[str, object], + litellm_params: dict[str, object], + headers: dict[str, str], + ) -> dict[str, object]: + return super().transform_request( + model=model, + messages=messages, + optional_params=filter_claude_platform_request_body( + optional_params, + unsupported_override=resolve_unsupported_override(litellm_params, optional_params=optional_params), + ), + litellm_params=litellm_params, + headers=headers, + ) + def get_model_response_iterator( self, streaming_response: Any, diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index f1066643874..d9fc813a594 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -112,6 +112,17 @@ def merge_bedrock_aws_request_params( return request_params +def s3_static_key_pair(params: Mapping[str, object]) -> tuple[str, str] | None: + """The s3_access_key_id / s3_secret_access_key pair when both are set, otherwise None.""" + s3_access_key_id: Final = params.get("s3_access_key_id") + s3_secret_access_key: Final = params.get("s3_secret_access_key") + if not isinstance(s3_access_key_id, str) or not s3_access_key_id: + return None + if not isinstance(s3_secret_access_key, str) or not s3_secret_access_key: + return None + return s3_access_key_id, s3_secret_access_key + + # Lazy import cache to avoid circular imports and performance impact _get_model_info = None @@ -776,7 +787,7 @@ def is_bedrock_application_inference_profile_arn(model: str) -> bool: def strip_bedrock_routing_prefix(model: str) -> str: """Strip LiteLLM routing prefixes from model name.""" - for prefix in ["bedrock/", "converse/", "invoke/", "openai/", "nova-2/", "nova/"]: + for prefix in ["bedrock/", "converse/", "invoke/", "openai/", "mantle/", "nova-2/", "nova/"]: if model.startswith(prefix): model = model.split("/", 1)[1] return model @@ -839,6 +850,7 @@ def get_bedrock_base_model(model: str) -> str: Handle model names like: - "us.meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1" - "bedrock/converse/model" -> "model" + - "bedrock/mantle/anthropic.claude-sonnet-5" -> "anthropic.claude-sonnet-5" - "anthropic.claude-3-5-sonnet-20241022-v2:0:51k" -> "anthropic.claude-3-5-sonnet-20241022-v2:0" - "bedrock/nova-2/arn:aws:..." -> "amazon.nova-2-custom" - "bedrock/nova/arn:aws:..." -> "amazon.nova-custom" diff --git a/litellm/llms/bedrock/files/handler.py b/litellm/llms/bedrock/files/handler.py index 0b75474ba1b..3d23b69f846 100644 --- a/litellm/llms/bedrock/files/handler.py +++ b/litellm/llms/bedrock/files/handler.py @@ -11,7 +11,6 @@ from litellm.litellm_core_utils.cloud_storage_security import ( validate_managed_cloud_file_id, ) from litellm.llms.custom_httpx.http_handler import get_async_httpx_client -from litellm.types.llms.bedrock import AwsAuthParams from litellm.types.llms.openai import ( FileContentRequest, HttpxBinaryResponseContent, @@ -103,9 +102,7 @@ class BedrockFilesHandler(BaseAWSLLM): ) aws_region_name: Final = self._get_aws_region_name(optional_params=optional_params, model="") - credentials: Final[Credentials] = self.resolve_credentials( - AwsAuthParams.model_validate(optional_params), aws_region_name - ) + credentials: Final[Credentials] = self.resolve_s3_credentials(optional_params, aws_region_name) # Create S3 client s3_client: Final = boto3.client( diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index a7486dd4de0..43faa7d79ea 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -63,7 +63,11 @@ from litellm.types.utils import ExtractedFileData, LlmProviders, SpecialEnums from litellm.utils import get_llm_provider from ..base_aws_llm import BaseAWSLLM -from ..common_utils import BedrockError, merge_bedrock_aws_request_params, resolve_s3_encryption_key_id +from ..common_utils import ( + BedrockError, + merge_bedrock_aws_request_params, + resolve_s3_encryption_key_id, +) S3_SIGNED_REQUEST_HEADERS_PARAM: Final = "_s3_signed_request_headers" @@ -148,6 +152,8 @@ class _BedrockS3RequestParams(AwsAuthParams): aws_region_name: str | None = None s3_region_name: str | None = None s3_endpoint_url: str | None = None + s3_access_key_id: str | None = None + s3_secret_access_key: str | None = None @dataclass(frozen=True, slots=True) @@ -1147,7 +1153,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") aws_region_name: Final = self._get_aws_region_name(optional_params=optional_params, model="") - credentials: Final = self.resolve_credentials(AwsAuthParams.model_validate(optional_params), aws_region_name) + credentials: Final = self.resolve_s3_credentials(optional_params, aws_region_name) # Calculate SHA256 hash of the content (REQUIRED for S3) content_hash: Final = hashlib.sha256(content.encode("utf-8")).hexdigest() @@ -1494,7 +1500,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): except ImportError: raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") - credentials: Final = self.resolve_credentials(request_params, aws_region_name) + credentials: Final = self.resolve_s3_credentials(request_params.model_dump(exclude_none=True), aws_region_name) empty_body_hash: Final = hashlib.sha256(b"").hexdigest() aws_request: Final = AWSRequest( # any-ok: botocore AWSRequest is untyped diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 1f37fafde01..f46edc766c7 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -81,6 +81,10 @@ class AmazonAnthropicClaudeMessagesConfig( def custom_llm_provider(self) -> str | None: return "bedrock" + @property + def beta_headers_provider(self) -> str: + return self.custom_llm_provider or "bedrock" + BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = frozenset(BedrockInvokeAnthropicMessagesRequest.__annotations__.keys()) def get_error_class( @@ -552,7 +556,7 @@ class AmazonAnthropicClaudeMessagesConfig( if "tool-search-tool-2025-10-19" in beta_set: beta_set.add("tool-examples-2025-10-29") - beta_provider: Final = self.custom_llm_provider or "bedrock" + beta_provider: Final = self.beta_headers_provider filtered_betas: Final = sorted( filter_and_transform_beta_headers( beta_headers=list(beta_set), diff --git a/litellm/llms/bedrock/messages/mantle_transformation.py b/litellm/llms/bedrock/messages/mantle_transformation.py index 7c8758960ad..052eb90a833 100644 --- a/litellm/llms/bedrock/messages/mantle_transformation.py +++ b/litellm/llms/bedrock/messages/mantle_transformation.py @@ -2,16 +2,20 @@ Transformation for Bedrock Mantle (Claude Mythos Preview) - /messages endpoint Inherits all Messages API request/response transformations from -AmazonAnthropicClaudeMessagesConfig. Overrides only the URL and model-prefix -stripping that are specific to the bedrock-mantle endpoint. +AmazonAnthropicClaudeMessagesConfig. Overrides the URL, the model-prefix +stripping, and the anthropic-version / anthropic-beta placement (headers, +never the body) that are specific to the bedrock-mantle endpoint. """ -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + DEFAULT_ANTHROPIC_API_VERSION, AnthropicMessagesConfig, ) from litellm.llms.bedrock.common_utils import build_mantle_messages_url @@ -31,6 +35,18 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_BODY_FIELDS_MANTLE_READS_FROM_HEADERS: Final = frozenset({"anthropic_version", "anthropic_beta"}) +_ANTHROPIC_BETAS: Final = TypeAdapter(tuple[str, ...]) +_MANTLE_REQUEST: Final = TypeAdapter(dict[str, object]) + + +def _move_betas_into_header(request: Mapping[str, object], headers: dict[str, str]) -> None: + betas: Final = _ANTHROPIC_BETAS.validate_python(request.get("anthropic_beta") or ()) + if betas: + headers["anthropic-beta"] = ",".join(betas) # rebind-ok: the handler signs and sends this same dict + return + headers.pop("anthropic-beta", None) # rebind-ok: a caller header Mantle rejects in full must not reach it + class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): """ @@ -40,6 +56,13 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): model ID in the request body (unlike Bedrock Invoke which puts it in the URL). """ + @property + def beta_headers_provider(self) -> str: + return "bedrock_mantle" + + def should_filter_anthropic_beta_headers(self) -> bool: + return False + def get_complete_url( self, api_base: str | None, @@ -66,7 +89,7 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): api_key: str | None = None, api_base: str | None = None, ) -> tuple[dict, str | None]: - headers, api_base = super().validate_anthropic_messages_environment( + merged_headers, resolved_api_base = super().validate_anthropic_messages_environment( headers=headers, model=model, messages=messages, @@ -76,9 +99,21 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): api_base=api_base, ) project_id: Final = litellm_params.get("aws_bedrock_project_id") - if project_id: - headers["anthropic-workspace"] = project_id - return headers, api_base + has_version: Final = any(name.lower() == "anthropic-version" for name in merged_headers) + mantle_headers: Final = MappingProxyType( + { + name: value + for name, value in ( + ("anthropic-workspace", project_id), + ("anthropic-version", None if has_version else DEFAULT_ANTHROPIC_API_VERSION), + ) + if value + } + ) + return { # mutable-ok: the base class contract returns a dict the handler signs into in place + **merged_headers, + **mantle_headers, + }, resolved_api_base def transform_anthropic_messages_request( self, @@ -88,25 +123,28 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig): litellm_params: GenericLiteLLMParams, headers: dict, ) -> dict: - # Strip "mantle/" routing prefix to get the real model ID model_id: Final = model.replace("mantle/", "", 1) - - request: Final = super().transform_anthropic_messages_request( - model=model_id, - messages=messages, - anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, - litellm_params=litellm_params, - headers=headers, + request: Final = _MANTLE_REQUEST.validate_python( + super().transform_anthropic_messages_request( + model=model_id, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ), ) - - # Parent (AmazonAnthropicClaudeMessagesConfig) removes "model" and - # "stream" from the body (Bedrock Invoke puts the model in the URL and - # streams via a dedicated endpoint). The mantle endpoint (Messages API) - # requires both in the request body. - stream_fields: Final[dict[str, bool]] = ( - {"stream": True} if anthropic_messages_optional_request_params.get("stream") is True else {} + _move_betas_into_header(request, headers) + body: Final = MappingProxyType( + {key: value for key, value in request.items() if key not in _BODY_FIELDS_MANTLE_READS_FROM_HEADERS} ) - return {**request, "model": model_id, **stream_fields} + streaming: Final = anthropic_messages_optional_request_params.get("stream") is True + mantle_fields: Final = MappingProxyType( + {key: value for key, value in (("model", model_id), ("stream", streaming)) if value} + ) + return { # mutable-ok: the base class contract returns the dict the handler serializes as the body + **body, + **mantle_fields, + } def transform_anthropic_messages_response( self, diff --git a/litellm/llms/bedrock_mantle/messages/transformation.py b/litellm/llms/bedrock_mantle/messages/transformation.py index 6e975d072ed..78881153e91 100644 --- a/litellm/llms/bedrock_mantle/messages/transformation.py +++ b/litellm/llms/bedrock_mantle/messages/transformation.py @@ -2,11 +2,6 @@ from collections.abc import Mapping from types import MappingProxyType from typing import Final -from pydantic import TypeAdapter - -from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( - DEFAULT_ANTHROPIC_API_VERSION, -) from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import MANTLE_MESSAGES_PATH from litellm.llms.bedrock.messages.mantle_transformation import AmazonMantleMessagesConfig @@ -17,7 +12,6 @@ from litellm.llms.bedrock_mantle.common_utils import ( ) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES -from litellm.types.router import GenericLiteLLMParams _BASE_SUFFIXES_TO_STRIP: Final = ( MANTLE_MESSAGES_PATH, @@ -27,9 +21,6 @@ _BASE_SUFFIXES_TO_STRIP: Final = ( "/openai/v1", "/v1", ) -_BODY_FIELDS_MANTLE_READS_FROM_HEADERS: Final = frozenset({"anthropic_version", "anthropic_beta"}) -_ANTHROPIC_BETAS: Final = TypeAdapter(tuple[str, ...]) -_MANTLE_REQUEST: Final = TypeAdapter(dict[str, object]) def build_mantle_native_messages_url(api_base: str | None, litellm_params: Mapping[str, object]) -> str: @@ -74,54 +65,3 @@ class BedrockMantleAnthropicMessagesConfig(BedrockMantleAuthMixin, AmazonMantleM stream: bool | None = None, ) -> str: return build_mantle_native_messages_url(api_base=api_base, litellm_params=litellm_params) - - def validate_anthropic_messages_environment( - self, - headers: dict, - model: str, - messages: list[dict], - optional_params: dict, - litellm_params: dict, - api_key: str | None = None, - api_base: str | None = None, - ) -> tuple[dict, str | None]: - merged_headers, resolved_api_base = super().validate_anthropic_messages_environment( - headers=headers, - model=model, - messages=messages, - optional_params=optional_params, - litellm_params=litellm_params, - api_key=api_key, - api_base=api_base, - ) - if any(name.lower() == "anthropic-version" for name in merged_headers): - return merged_headers, resolved_api_base - return { # mutable-ok: the base class contract returns a dict the handler signs into in place - **merged_headers, - "anthropic-version": DEFAULT_ANTHROPIC_API_VERSION, - }, resolved_api_base - - def transform_anthropic_messages_request( - self, - model: str, - messages: list[dict], - anthropic_messages_optional_request_params: dict, - litellm_params: GenericLiteLLMParams, - headers: dict, - ) -> dict: - request: Final = _MANTLE_REQUEST.validate_python( - super().transform_anthropic_messages_request( - model=model, - messages=messages, - anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, - litellm_params=litellm_params, - headers=headers, - ), - ) - betas: Final = request.get("anthropic_beta") - if betas is not None: - header_betas: Final = ",".join(_ANTHROPIC_BETAS.validate_python(betas)) - headers["anthropic-beta"] = header_betas # rebind-ok: the handler signs and sends this same dict - return { # mutable-ok: the base class contract returns the dict the handler serializes as the body - key: value for key, value in request.items() if key not in _BODY_FIELDS_MANTLE_READS_FROM_HEADERS - } diff --git a/litellm/llms/black_forest_labs/image_generation/transformation.py b/litellm/llms/black_forest_labs/image_generation/transformation.py index 119ffff1c34..e5c2bdf59e9 100644 --- a/litellm/llms/black_forest_labs/image_generation/transformation.py +++ b/litellm/llms/black_forest_labs/image_generation/transformation.py @@ -29,9 +29,8 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -258,7 +257,7 @@ class BlackForestLabsImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/brave/search/__init__.py b/litellm/llms/brave/search/__init__.py index cc1168d7ef8..de70c62e040 100644 --- a/litellm/llms/brave/search/__init__.py +++ b/litellm/llms/brave/search/__init__.py @@ -1,7 +1,7 @@ -""" -Brave Search API module. -""" - -from litellm.llms.brave.search.transformation import BraveSearchConfig - -__all__ = ["BraveSearchConfig"] +""" +Brave Search API module. +""" + +from litellm.llms.brave.search.transformation import BraveSearchConfig + +__all__ = ["BraveSearchConfig"] diff --git a/litellm/llms/bytez/chat/transformation.py b/litellm/llms/bytez/chat/transformation.py index 7977db0f056..e622761dd7f 100644 --- a/litellm/llms/bytez/chat/transformation.py +++ b/litellm/llms/bytez/chat/transformation.py @@ -23,9 +23,8 @@ from litellm.utils import CustomStreamWrapper, ModelResponse, Usage from ..common_utils import API_BASE, BytezError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -187,7 +186,7 @@ class BytezChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/clarifai/chat/transformation.py b/litellm/llms/clarifai/chat/transformation.py index 76d35467497..a0946254de0 100644 --- a/litellm/llms/clarifai/chat/transformation.py +++ b/litellm/llms/clarifai/chat/transformation.py @@ -13,9 +13,8 @@ from litellm.types.utils import ModelResponse from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -87,7 +86,7 @@ class ClarifaiConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/cohere/chat/transformation.py b/litellm/llms/cohere/chat/transformation.py index fa46bd7f6cf..a26cdc81695 100644 --- a/litellm/llms/cohere/chat/transformation.py +++ b/litellm/llms/cohere/chat/transformation.py @@ -15,9 +15,8 @@ from ..common_utils import ModelResponseIterator as CohereModelResponseIterator from ..common_utils import validate_environment as cohere_validate_environment if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -227,7 +226,7 @@ class CohereChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/cohere/chat/v2_transformation.py b/litellm/llms/cohere/chat/v2_transformation.py index 4252e7d02e9..37c53640d18 100644 --- a/litellm/llms/cohere/chat/v2_transformation.py +++ b/litellm/llms/cohere/chat/v2_transformation.py @@ -20,9 +20,8 @@ from ..common_utils import CohereError, CohereV2ModelResponseIterator from ..common_utils import validate_environment as cohere_validate_environment if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -191,7 +190,7 @@ class CohereV2ChatConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/cohere/embed/handler.py b/litellm/llms/cohere/embed/handler.py index 3cebf6b9a90..6d9543d7c30 100644 --- a/litellm/llms/cohere/embed/handler.py +++ b/litellm/llms/cohere/embed/handler.py @@ -20,7 +20,7 @@ from litellm.types.utils import EmbeddingResponse from .v1_transformation import CohereEmbeddingConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer def validate_environment(api_key, headers: dict): @@ -60,7 +60,7 @@ async def async_embedding( api_base: str, api_key: str | None, headers: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", client: AsyncHTTPHandler | None = None, ): ## LOGGING @@ -122,7 +122,7 @@ def embedding( logging_obj: LiteLLMLoggingObj, optional_params: dict, headers: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", data: dict | CohereEmbeddingRequest | None = None, complete_api_base: str | None = None, api_key: str | None = None, diff --git a/litellm/llms/cometapi/image_generation/cost_calculator.py b/litellm/llms/cometapi/image_generation/cost_calculator.py index 0ad9f75c45f..8f767cf311e 100644 --- a/litellm/llms/cometapi/image_generation/cost_calculator.py +++ b/litellm/llms/cometapi/image_generation/cost_calculator.py @@ -1,19 +1,22 @@ from typing import Any, Final import litellm -from litellm.types.utils import ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.types.utils import ImageResponse, ModelInfo def cost_calculator( model: str, image_response: Any, + model_info: ModelInfo | None = None, ) -> float: """ CometAPI image generation cost calculator """ - _model_info: Final = litellm.get_model_info( + _model_info: Final = resolve_image_model_info( model=model, custom_llm_provider=litellm.LlmProviders.COMETAPI.value, + model_info=model_info, ) output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 diff --git a/litellm/llms/cometapi/image_generation/transformation.py b/litellm/llms/cometapi/image_generation/transformation.py index 03c820de198..4432c151a64 100644 --- a/litellm/llms/cometapi/image_generation/transformation.py +++ b/litellm/llms/cometapi/image_generation/transformation.py @@ -13,9 +13,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -132,7 +131,7 @@ class CometAPIImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/compactifai/chat/transformation.py b/litellm/llms/compactifai/chat/transformation.py index 3c5a889ce63..a02db2338d8 100644 --- a/litellm/llms/compactifai/chat/transformation.py +++ b/litellm/llms/compactifai/chat/transformation.py @@ -17,9 +17,8 @@ from litellm.types.utils import ModelResponse from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -66,7 +65,7 @@ class CompactifAIChatConfig(OpenAIGPTConfig): messages: Sequence[AllMessageValues], optional_params: Mapping[str, object], litellm_params: Mapping[str, object], - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/compaction.py b/litellm/llms/compaction.py new file mode 100644 index 00000000000..16afa2e2c25 --- /dev/null +++ b/litellm/llms/compaction.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias + +from pydantic import TypeAdapter + +from litellm.types.llms.openai import AllMessageValues + +if TYPE_CHECKING: + from litellm.router import Router + +CompactionProtocol: TypeAlias = Literal["chat", "messages"] +_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_MESSAGES: Final = TypeAdapter(list[AllMessageValues]) + + +class NativeCompactionProvider(Protocol): + def supports_native_compaction(self, params: Mapping[str, object]) -> bool: ... + + def compatible_defaults(self, payload: Mapping[str, object]) -> bool: ... + + def request_kwargs(self) -> Mapping[str, object]: ... + + def extract_summary(self, protocol: CompactionProtocol, response: Mapping[str, object]) -> str | None: ... + + +def get_native_compaction_provider(params: Mapping[str, object]) -> NativeCompactionProvider | None: + from litellm.llms.anthropic import compaction + + return compaction if compaction.supports_native_compaction(params) else None + + +async def dispatch(router: Router, protocol: CompactionProtocol, payload: Mapping[str, object]) -> Mapping[str, object]: + if protocol == "messages": + return _MAPPING.validate_python( + await router.aanthropic_messages(custom_llm_provider=None, client=None, **payload) + ) + response: Final = await router.acompletion( + model=str(payload["model"]), + messages=_MESSAGES.validate_python(payload["messages"]), + stream=False, + **MappingProxyType( + {key: value for key, value in payload.items() if key not in ("model", "messages", "stream")} + ), + ) + return _MAPPING.validate_python(response.model_dump()) diff --git a/litellm/llms/custom_httpx/aiohttp_handler.py b/litellm/llms/custom_httpx/aiohttp_handler.py index 0809ef5274f..034c9514092 100644 --- a/litellm/llms/custom_httpx/aiohttp_handler.py +++ b/litellm/llms/custom_httpx/aiohttp_handler.py @@ -26,9 +26,8 @@ from litellm.types.utils import HttpHandlerRequestFields, ImageResponse, LlmProv from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -268,7 +267,7 @@ class BaseLLMAIOHTTPHandler: messages: list, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, client: ClientSession | None = None, ): diff --git a/litellm/llms/custom_httpx/asgi_handler.py b/litellm/llms/custom_httpx/asgi_handler.py new file mode 100644 index 00000000000..ab704ab12a6 --- /dev/null +++ b/litellm/llms/custom_httpx/asgi_handler.py @@ -0,0 +1,46 @@ +from collections.abc import Generator +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +from typing import Final + +import httpx +from starlette.types import ASGIApp, Receive, Scope, Send + +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared cache retains its legacy parameter mapping +) +from litellm.types.llms.custom_http import httpxSpecialProvider + + +@dataclass(frozen=True, slots=True) +class _ASGITarget: + app: ASGIApp + root_path: str + client: tuple[str, int] | None + + +_target: Final[ContextVar[_ASGITarget]] = ContextVar("httpx_asgi_target") + + +async def _dispatch(scope: Scope, receive: Receive, send: Send) -> None: + target: Final = _target.get() + await target.app({**scope, "root_path": target.root_path, "client": target.client}, receive, send) + + +_TRANSPORT: Final = httpx.ASGITransport(app=_dispatch, raise_app_exceptions=False) + + +@contextmanager +def get_async_asgi_client( + app: ASGIApp, root_path: str = "", client: tuple[str, int] | None = None +) -> Generator[httpx.AsyncClient]: + handler: Final = get_async_httpx_client( + llm_provider=httpxSpecialProvider.ASGI, + params={"transport": _TRANSPORT, "timeout": httpx.Timeout(None), "follow_redirects": False}, + ) + token: Final = _target.set(_ASGITarget(app, root_path, client)) + try: + yield handler.client + finally: + _target.reset(token) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index b7b2477e85c..312f46bb021 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -614,11 +614,15 @@ class AsyncHTTPHandler: client_alias: str | None = None, # name for client in logs ssl_verify: VerifyTypes | None = None, shared_session: Optional["ClientSession"] = None, + transport: httpx.AsyncBaseTransport | None = None, + follow_redirects: bool = True, ): self.timeout = timeout self.event_hooks = event_hooks self.ssl_verify = ssl_verify self.shared_session = shared_session + self.transport = transport + self.follow_redirects = follow_redirects self._owns_client = True self._client = self.create_client( timeout=timeout, @@ -651,6 +655,16 @@ class AsyncHTTPHandler: ssl_verify: VerifyTypes | None = None, shared_session: Optional["ClientSession"] = None, ) -> httpx.AsyncClient: + if self.transport is not None: + return httpx.AsyncClient( + transport=self.transport, + event_hooks=event_hooks, + timeout=timeout if timeout is not None else _DEFAULT_TIMEOUT, + headers=get_default_headers(), + cookies=blocked_cookie_jar(), + follow_redirects=self.follow_redirects, + trust_env=False, + ) # Get unified SSL configuration ssl_config: Final = get_ssl_configuration(ssl_verify) @@ -680,7 +694,7 @@ class AsyncHTTPHandler: cert=cert, headers=default_headers, cookies=blocked_cookie_jar(), - follow_redirects=True, + follow_redirects=self.follow_redirects, http2=http2_enabled(), ) @@ -831,6 +845,7 @@ class AsyncHTTPHandler: params=params, headers=headers, stream=stream, + content=content, ) finally: await new_client.aclose() @@ -971,6 +986,7 @@ class AsyncHTTPHandler: params=params, headers=headers, stream=stream, + content=content, ) finally: await new_client.aclose() @@ -1037,6 +1053,7 @@ class AsyncHTTPHandler: params=params, headers=headers, stream=stream, + content=content, ) finally: await new_client.aclose() @@ -1694,7 +1711,7 @@ class HTTPHandler: def get_async_httpx_client( - llm_provider: LlmProviders | httpxSpecialProvider, + llm_provider: LlmProviders | httpxSpecialProvider | str, params: dict | None = None, shared_session: Optional["ClientSession"] = None, ) -> AsyncHTTPHandler: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index db821f42a90..8cbc28362a8 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -4,7 +4,6 @@ import ssl from collections.abc import AsyncGenerator, AsyncIterator, Coroutine, Iterator, Mapping, Sequence from contextlib import asynccontextmanager from functools import lru_cache -from itertools import chain from types import MappingProxyType, ModuleType from typing import ( TYPE_CHECKING, @@ -34,6 +33,7 @@ from litellm._logging import _redact_string, verbose_logger from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.files.types import FileContentStreamingResult +from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, @@ -45,7 +45,11 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( ) from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields -from litellm.litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason +from litellm.litellm_core_utils.realtime_errors import ( + close_after_upstream_handshake_refusal, + realtime_error_event, + websocket_close_reason, +) from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.anthropic_messages.transformation import ( @@ -182,22 +186,22 @@ from litellm.utils import ( def _rust_responses_websocket_enabled( custom_llm_provider: str | None, ) -> bool: - from litellm.rust_bridge.catalog import Context, Delivery, Route, decision + from litellm.rust_bridge.catalog import Delivery, Route, RouteContext, decision from litellm.rust_bridge.configuration import Decision - context: Final = Context(Route.RESPONSES, provider=custom_llm_provider, delivery=Delivery.WEBSOCKET) + context: Final = RouteContext(Route.RESPONSES, provider=custom_llm_provider, delivery=Delivery.WEBSOCKET) return decision(context) is not Decision.PYTHON from .http_handler import get_shared_realtime_ssl_context if TYPE_CHECKING: - import tiktoken from aiohttp import ClientSession from websockets.asyncio.client import ClientConnection from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) @@ -493,7 +497,7 @@ class BaseLLMHTTPHandler: messages: list, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, client: AsyncHTTPHandler | None = None, json_mode: bool = False, @@ -559,7 +563,7 @@ class BaseLLMHTTPHandler: api_base: str | None, custom_llm_provider: str, model_response: ModelResponse, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", logging_obj: LiteLLMLoggingObj, optional_params: dict, timeout: float | httpx.Timeout, @@ -2853,7 +2857,7 @@ class BaseLLMHTTPHandler: id(shared_session) if shared_session else None, ) async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, shared_session=shared_session, ) @@ -5614,28 +5618,22 @@ class BaseLLMHTTPHandler: } internal_keys: Final = {"litellm_logging_obj"} - kwargs_for_followup: Final = MappingProxyType( - { - key: value - for key, value in chain( - ( - (k, v) - for k, v in kwargs.items() - if not is_interception_internal_key( - k, prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES - ) - and k != "_code_interpreter_interception_converted_stream" - and k not in internal_keys - and k not in optional_params - ), - ((k, v) for k, v in patch.kwargs.items() if k not in optional_params), - ( - ("_agentic_loop_depth", depth + 1), - ("max_agentic_loops", max_loops), - ("_agentic_loop_fingerprints", fingerprints + [fingerprint]), - ), - ) - } + kwargs_for_followup: Final = build_agentic_followup_kwargs( + request_kwargs=MappingProxyType( + { + k: v + for k, v in kwargs.items() + if not is_interception_internal_key(k, prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES) + and k != "_code_interpreter_interception_converted_stream" + and k not in internal_keys + } + ), + patch_kwargs=patch.kwargs, + request_params=frozenset((*optional_params, "model", "input")), + depth=depth, + max_loops=max_loops, + fingerprints=fingerprints, + fingerprint=fingerprint, ) try: @@ -5756,17 +5754,23 @@ class BaseLLMHTTPHandler: "stream_response", "custom_prompt_dict", } - kwargs_for_followup: Final = { - k: v - for k, v in kwargs.items() - if not k.startswith("_websearch_interception") - and not k.startswith("_compression_interception") - and k not in internal_params - } - kwargs_for_followup.update(patch.kwargs) - kwargs_for_followup["_agentic_loop_depth"] = depth + 1 - kwargs_for_followup["max_agentic_loops"] = max_loops - kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint] + kwargs_for_followup: Final = build_agentic_followup_kwargs( + request_kwargs=MappingProxyType( + { + k: v + for k, v in kwargs.items() + if not k.startswith("_websearch_interception") + and not k.startswith("_compression_interception") + and k not in internal_params + } + ), + patch_kwargs=patch.kwargs, + request_params=frozenset((*optional_params_for_followup, "model", "messages")), + depth=depth, + max_loops=max_loops, + fingerprints=fingerprints, + fingerprint=fingerprint, + ) return await litellm.acompletion( model=full_model_name, @@ -6384,9 +6388,9 @@ class BaseLLMHTTPHandler: await realtime_streaming.bidirectional_forward() - except websockets.exceptions.InvalidStatusCode as e: + except websockets.exceptions.InvalidStatus as e: verbose_logger.exception("Error connecting to backend: %s", e) - await websocket.close(code=e.status_code, reason=_redact_string(str(e))) + await close_after_upstream_handshake_refusal(websocket, e.response.status_code) except Exception as e: verbose_logger.exception("Error connecting to backend: %s", e) redacted_error: Final = _redact_string(str(e)) @@ -6799,9 +6803,9 @@ class BaseLLMHTTPHandler: ) return await streaming.bidirectional_forward() - except websockets.exceptions.InvalidStatusCode as e: + except websockets.exceptions.InvalidStatus as e: verbose_logger.exception("Error connecting to responses WS backend: %s", e) - await websocket.close(code=e.status_code, reason=_redact_string(str(e))) + await close_after_upstream_handshake_refusal(websocket, e.response.status_code) except Exception as e: verbose_logger.exception("Error in responses WS: %s", e) try: @@ -8808,6 +8812,7 @@ class BaseLLMHTTPHandler: raw_response=response, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, + client=sync_httpx_client, ) except Exception as e: @@ -8897,6 +8902,7 @@ class BaseLLMHTTPHandler: raw_response=response, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, + client=async_httpx_client, ) except Exception as e: diff --git a/litellm/llms/dashscope/image_generation/transformation.py b/litellm/llms/dashscope/image_generation/transformation.py index c0e278a96ef..ffa60a3d9bc 100644 --- a/litellm/llms/dashscope/image_generation/transformation.py +++ b/litellm/llms/dashscope/image_generation/transformation.py @@ -38,9 +38,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -165,7 +164,7 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index dd257cd68b0..538904b34e6 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -16,6 +16,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( _extract_reasoning_content, # pyright: ignore[reportPrivateUsage] # same import as the OpenAI transformation + merge_consecutive_system_messages, strip_litellm_internal_message_fields, strip_name_from_message, ) @@ -148,9 +149,8 @@ def _split_parallel_tool_calls(messages: list[AllMessageValues]) -> list[AllMess if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -188,7 +188,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): return "databricks" @classmethod - def get_config(cls): + def get_config(cls, *, model: str | None = None): return super().get_config() def get_required_params(self) -> list[ProviderField]: @@ -465,7 +465,9 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): new_messages.append(_message) if "claude" not in model: - new_messages = _split_parallel_tool_calls(cast(list[AllMessageValues], new_messages)) + new_messages = _split_parallel_tool_calls( + merge_consecutive_system_messages(cast(list[AllMessageValues], new_messages)) + ) if is_async: return super()._transform_messages(messages=new_messages, model=model, is_async=cast(Literal[True], True)) @@ -648,7 +650,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/deprecated_providers/aleph_alpha.py b/litellm/llms/deprecated_providers/aleph_alpha.py index 2ad9ce4edc8..8f5c35f32f2 100644 --- a/litellm/llms/deprecated_providers/aleph_alpha.py +++ b/litellm/llms/deprecated_providers/aleph_alpha.py @@ -277,12 +277,7 @@ def completion( ## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here. prompt_tokens: Final = len(encoding.encode(prompt)) - completion_tokens: Final = len( - encoding.encode( - model_response["choices"][0]["message"]["content"], - disallowed_special=(), - ) - ) + completion_tokens: Final = len(encoding.encode(model_response["choices"][0]["message"]["content"])) model_response.created = int(time.time()) model_response.model = model diff --git a/litellm/llms/edenai/chat/transformation.py b/litellm/llms/edenai/chat/transformation.py index 4fd5a9d550b..67d308d9e38 100644 --- a/litellm/llms/edenai/chat/transformation.py +++ b/litellm/llms/edenai/chat/transformation.py @@ -25,9 +25,8 @@ from litellm.types.utils import ModelResponse, ModelResponseStream, Usage from ..common_utils import EdenAIException, reported_cost, resolve_api_base, resolve_api_key if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding _OPTIONAL_MAPPING: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) @@ -97,7 +96,7 @@ class EdenAIChatConfig(OpenAIGPTConfig): messages: list[AllMessageValues], # mutable-ok: inherited contract optional_params: dict[str, object], # mutable-ok: inherited contract litellm_params: dict[str, object], # mutable-ok: inherited contract - encoding: "tiktoken.Encoding | None", + encoding: "Encoding | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/edenai/image_generation/transformation.py b/litellm/llms/edenai/image_generation/transformation.py index 2965c4041de..7f729cd7fbb 100644 --- a/litellm/llms/edenai/image_generation/transformation.py +++ b/litellm/llms/edenai/image_generation/transformation.py @@ -19,9 +19,8 @@ from litellm.utils import convert_to_model_response_object from ..common_utils import EdenAIException, endpoint_url, json_headers, pick, reported_cost if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding _SUPPORTED_PARAMS: Final[tuple[OpenAIImageGenerationOptionalParams, ...]] = ( "background", @@ -94,7 +93,7 @@ class EdenAIImageGenerationConfig(BaseImageGenerationConfig): request_data: dict[str, object], # mutable-ok: inherited contract optional_params: dict[str, object], # mutable-ok: inherited contract litellm_params: dict[str, object], # mutable-ok: inherited contract - encoding: "tiktoken.Encoding | None", + encoding: "Encoding | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/edenai/videos/transformation.py b/litellm/llms/edenai/videos/transformation.py index 25c7bcf24ea..31572e9e5fe 100644 --- a/litellm/llms/edenai/videos/transformation.py +++ b/litellm/llms/edenai/videos/transformation.py @@ -21,6 +21,7 @@ from ..common_utils import EdenAIException, authorized_headers, endpoint_url, re if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.custom_httpx.http_handler import HTTPHandler def _usage_with_reported_cost( @@ -110,6 +111,7 @@ class EdenAIVideoConfig(OpenAIVideoConfig): raw_response: httpx.Response, logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str | None = None, + client: "HTTPHandler | None" = None, ) -> VideoObject: raw_response.raise_for_status() # the shared GET helpers return error bodies instead of raising video: Final = super().transform_video_status_retrieve_response( diff --git a/litellm/llms/fal_ai/chat/__init__.py b/litellm/llms/fal_ai/chat/__init__.py new file mode 100644 index 00000000000..b2a4a006aef --- /dev/null +++ b/litellm/llms/fal_ai/chat/__init__.py @@ -0,0 +1,3 @@ +from .transformation import FalAIChatConfig, FalAIError + +__all__ = ("FalAIChatConfig", "FalAIError") diff --git a/litellm/llms/fal_ai/chat/transformation.py b/litellm/llms/fal_ai/chat/transformation.py new file mode 100644 index 00000000000..164b660b21a --- /dev/null +++ b/litellm/llms/fal_ai/chat/transformation.py @@ -0,0 +1,244 @@ +""" +Support for `/v1/chat/completions` on Fal AI model endpoints, e.g. fal-ai/moondream3-preview/query. + +These endpoints are not OpenAI-compatible: the request body is a flat ``{"prompt", "image_url"}`` +object and the response is ``{"output", "reasoning", "finish_reason", "usage_info"}``. +""" + +import time +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Final + +import httpx +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import Message, ModelResponse, Usage + +if TYPE_CHECKING: + import tiktoken + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +DEFAULT_BASE_URL: Final[str] = "https://fal.run" +PROVIDER_PREFIX: Final[str] = "fal_ai/" +PASSTHROUGH_PARAMS: Final[frozenset[str]] = frozenset(("reasoning", "temperature", "top_p")) +REASONING_DISABLED_EFFORTS: Final[frozenset[str]] = frozenset(("none", "minimal")) +REASONING_ENABLED_EFFORTS: Final[frozenset[str]] = frozenset(("low", "medium", "high")) + + +class _FalUsage(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + input_tokens: int + output_tokens: int + + +class _FalChatResponse(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + output: str + usage_info: _FalUsage + reasoning: str | None = None + finish_reason: str | None = None + + +_CHAT_RESPONSE: Final = TypeAdapter(_FalChatResponse) + + +class FalAIError(BaseLLMException): + def __init__( + self, + status_code: int, + message: str, + headers: dict | httpx.Headers | None = None, # mutable-ok: BaseLLMException header contract + ) -> None: + super().__init__(status_code=status_code, message=message, headers=headers) + + +def _image_part_url(part: Mapping[str, object]) -> str | None: + image_url: Final = part.get("image_url") + if isinstance(image_url, str): + return image_url + if isinstance(image_url, Mapping): + url: Final = image_url.get("url") + return url if isinstance(url, str) else None + return None + + +def _prompt_and_image(messages: Sequence[AllMessageValues]) -> tuple[str, str]: + if len(messages) != 1 or messages[0].get("role") != "user": + raise FalAIError( + status_code=400, + message="fal_ai chat completions accept exactly one user message; system prompts and multi-turn history are not supported", + ) + content: Final = messages[0].get("content") + if isinstance(content, str): + if not content: + raise FalAIError(status_code=400, message="fal_ai chat completions require text in the user message") + raise FalAIError( + status_code=400, + message="fal_ai chat completions require exactly one image_url content part in the user message", + ) + parts: Final[tuple[Mapping[str, object], ...]] = ( + tuple(part for part in content if isinstance(part, Mapping)) if isinstance(content, Sequence) else () + ) + prompt: Final = "\n".join( + text for part in parts if part.get("type") == "text" and isinstance((text := part.get("text")), str) and text + ) + image_urls: Final = tuple( + url for part in parts if part.get("type") == "image_url" and (url := _image_part_url(part)) is not None + ) + if not prompt: + raise FalAIError(status_code=400, message="fal_ai chat completions require text in the user message") + if len(image_urls) != 1: + raise FalAIError( + status_code=400, + message="fal_ai chat completions require exactly one image_url content part in the user message", + ) + return prompt, image_urls[0] + + +class FalAIChatConfig(BaseConfig): + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return api_key or get_secret_str("FAL_AI_API_KEY") + + @staticmethod + def get_api_base(api_base: str | None = None) -> str: + return (api_base or get_secret_str("FAL_AI_API_BASE") or DEFAULT_BASE_URL).rstrip("/") + + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: inherited contract returns a list + return list(("reasoning_effort", "temperature", "top_p")) # mutable-ok: inherited contract returns a list + + def _map_reasoning_effort(self, value: object, model: str, drop_params: bool) -> bool | None: + if isinstance(value, str) and value in REASONING_DISABLED_EFFORTS: + return False + if isinstance(value, str) and value in REASONING_ENABLED_EFFORTS: + return True + if drop_params: + return None + raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}") + + def _translate_param(self, param: str, value: object, model: str, drop_params: bool) -> tuple[str, object] | None: + if param in ("temperature", "top_p"): + return param, value + if param == "reasoning_effort": + reasoning: Final = self._map_reasoning_effort(value, model, drop_params) + return ("reasoning", reasoning) if reasoning is not None else None + return None + + def map_openai_params( + self, + non_default_params: dict, # mutable-ok: inherited contract + optional_params: dict, # mutable-ok: inherited contract + model: str, + drop_params: bool, + ) -> dict: # mutable-ok: inherited contract returns a dict + mapped: Final = { # mutable-ok: intermediate translation map, folded into the returned dict + translated[0]: translated[1] + for param, value in non_default_params.items() + if (translated := self._translate_param(param, value, model, drop_params)) is not None + } + return {**optional_params, **mapped} # mutable-ok: inherited contract returns a dict + + def validate_environment( + self, + headers: dict, # mutable-ok: inherited contract + model: str, + messages: list[AllMessageValues], # mutable-ok: inherited contract + optional_params: dict, # mutable-ok: inherited contract + litellm_params: dict, # mutable-ok: inherited contract + api_key: str | None = None, + api_base: str | None = None, + ) -> dict: # mutable-ok: inherited contract returns a dict + final_api_key: Final = self.get_api_key(api_key) + if not final_api_key: + raise ValueError("FAL_AI_API_KEY is not set") + return { # mutable-ok: inherited contract returns a dict + "content-type": "application/json", + **(headers or {}), # mutable-ok: empty default for the inherited contract's headers + "Authorization": f"Key {final_api_key}", + } + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: dict, # mutable-ok: inherited contract + litellm_params: dict, # mutable-ok: inherited contract + stream: bool | None = None, + ) -> str: + return f"{self.get_api_base(api_base)}/{model.removeprefix(PROVIDER_PREFIX)}" + + def transform_request( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: inherited contract + optional_params: dict, # mutable-ok: inherited contract + litellm_params: dict, # mutable-ok: inherited contract + headers: dict, # mutable-ok: inherited contract + ) -> dict: # mutable-ok: inherited contract returns a dict + if optional_params.get("stream"): + raise FalAIError(status_code=400, message="fal_ai chat completions do not support streaming") + prompt, image_url = _prompt_and_image(messages) + return { # mutable-ok: JSON request body + "prompt": prompt, + "image_url": image_url, + **{ # mutable-ok: JSON request body + key: value for key, value in optional_params.items() if key in PASSTHROUGH_PARAMS and value is not None + }, + } + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: "LiteLLMLoggingObj", + request_data: dict, # mutable-ok: inherited contract + messages: list[AllMessageValues], # mutable-ok: inherited contract + optional_params: dict, # mutable-ok: inherited contract + litellm_params: dict, # mutable-ok: inherited contract + encoding: "tiktoken.Encoding | None", + api_key: str | None = None, + json_mode: bool | None = None, + ) -> ModelResponse: + try: + completion_response: Final = _CHAT_RESPONSE.validate_json(raw_response.content) + except ValueError: + raise FalAIError( + status_code=422, + message=f"fal_ai returned an unexpected response body: {raw_response.text}", + headers=raw_response.headers, + ) + + message: Final = Message( + content=completion_response.output, + role="assistant", + reasoning_content=completion_response.reasoning, + ) + model_response.choices[0].message = message # rebind-ok: ModelResponse populated in place per contract + model_response.choices[0].finish_reason = map_finish_reason( # rebind-ok: same contract + completion_response.finish_reason or "stop" + ) + model_response.created = int(time.time()) # rebind-ok: same contract + model_response.model = model # rebind-ok: same contract + model_response.usage = Usage( # rebind-ok: same contract + prompt_tokens=completion_response.usage_info.input_tokens, + completion_tokens=completion_response.usage_info.output_tokens, + total_tokens=completion_response.usage_info.input_tokens + completion_response.usage_info.output_tokens, + ) + return model_response + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict | httpx.Headers, # mutable-ok: inherited contract + ) -> BaseLLMException: + return FalAIError(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/llms/fal_ai/cost_calculator.py b/litellm/llms/fal_ai/cost_calculator.py index fd7d82d314f..519a11de13c 100644 --- a/litellm/llms/fal_ai/cost_calculator.py +++ b/litellm/llms/fal_ai/cost_calculator.py @@ -1,3 +1,4 @@ +import os from collections.abc import Mapping from math import ceil from types import MappingProxyType @@ -6,10 +7,12 @@ from typing import Final from pydantic import TypeAdapter import litellm -from litellm.types.utils import ImageObject, ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import deployment_pricing, resolve_image_model_info +from litellm.types.utils import ImageObject, ImageResponse, ModelInfo FAL_KEYED_PRICING_DEFAULT_QUALITY: Final[str] = "high" -FAL_TEXT_TO_IMAGE_DEFAULT_SIZE: Final[str] = "1024-x-768" +_DEFAULT_KEYED_DIMENSIONS: Final[tuple[int, int]] = (1024, 768) +FAL_TEXT_TO_IMAGE_DEFAULT_SIZE: Final[str] = f"{_DEFAULT_KEYED_DIMENSIONS[0]}-x-{_DEFAULT_KEYED_DIMENSIONS[1]}" FAL_PIXELS_PER_MEGAPIXEL: Final[int] = 1_048_576 FAL_NAMED_IMAGE_SIZES: Final[Mapping[str, str]] = MappingProxyType( { @@ -24,6 +27,12 @@ FAL_NAMED_IMAGE_SIZES: Final[Mapping[str, str]] = MappingProxyType( _OBJECT_MAP: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +FAL_AI_QUEUE_DEFAULT_BASE: Final[str] = "https://queue.fal.run" + + +def fal_ai_queue_base() -> str: + return os.getenv("FAL_AI_QUEUE_API_BASE") or FAL_AI_QUEUE_DEFAULT_BASE + def _keyed_size(optional_params: Mapping[str, object]) -> str | None: image_size: Final = optional_params.get("image_size") @@ -55,37 +64,56 @@ def _image_dimensions(image: object) -> tuple[int, int] | None: return width, height -def _response_size(image: object) -> str | None: - dimensions: Final = _image_dimensions(image) - if dimensions is None: - return None - width, height = dimensions - return f"{width}-x-{height}" - - def _keyed_quality(optional_params: Mapping[str, object]) -> str: raw_quality: Final = optional_params.get("quality") return raw_quality if isinstance(raw_quality, str) and raw_quality != "auto" else FAL_KEYED_PRICING_DEFAULT_QUALITY +def _parse_keyed_dimensions(size: str | None) -> tuple[int, int] | None: + if size is None: + return None + parts: Final = tuple(size.split("-x-")) + if len(parts) != 2: + return None + try: + width, height = (int(part) for part in parts) + except ValueError: + return None + return (width, height) if width > 0 and height > 0 else None + + +def _keyed_rows(model: str, quality: str) -> tuple[tuple[int, int, float], ...]: + prefix: Final = f"fal_ai/{quality}/" + suffix: Final = f"/{model}" + return tuple( + (width, height, float(raw_cost)) + for key in litellm.model_cost + if isinstance(key, str) and key.startswith(prefix) and key.endswith(suffix) + for size in (key[len(prefix) : -len(suffix)],) + for dimensions in (_parse_keyed_dimensions(size),) + if dimensions is not None + for entry in (_entry(key),) + if entry is not None + for raw_cost in (entry.get("output_cost_per_image"),) + if isinstance(raw_cost, (int, float)) + for width, height in (dimensions,) + ) + + def _keyed_cost_per_image( model: str, image: object, optional_params: Mapping[str, object], ) -> float | None: quality: Final = _keyed_quality(optional_params) - request_size: Final = _keyed_size(optional_params) or FAL_TEXT_TO_IMAGE_DEFAULT_SIZE - sizes: Final = (_response_size(image), request_size, FAL_TEXT_TO_IMAGE_DEFAULT_SIZE) - for size in sizes: - if size is None: - continue - keyed_entry = _entry(f"fal_ai/{quality}/{size}/{model}") - if keyed_entry is None: - continue - keyed_cost = keyed_entry.get("output_cost_per_image") - if isinstance(keyed_cost, (int, float)): - return float(keyed_cost) - return None + rows: Final = _keyed_rows(model, quality) + if not rows: + return None + target_dimensions: Final = ( + _image_dimensions(image) or _parse_keyed_dimensions(_keyed_size(optional_params)) or _DEFAULT_KEYED_DIMENSIONS + ) + target_pixels: Final = target_dimensions[0] * target_dimensions[1] + return min(rows, key=lambda row: (abs(row[0] * row[1] - target_pixels), row[0] * row[1]))[2] def _flat_cost_per_image( @@ -108,10 +136,27 @@ def _entry(key: str) -> Mapping[str, object] | None: return _OBJECT_MAP.validate_python(raw_entry) +def _resolution_key(resolution: object) -> str | None: + if isinstance(resolution, bool) or not isinstance(resolution, (int, str)): + return None + return str(resolution) + + +def fal_ai_passthrough_cost(model: str, request_body: Mapping[str, object]) -> float | None: + entry: Final = _entry(f"{litellm.LlmProviders.FAL_AI.value}/{model}") + if entry is None: + return None + resolution: Final = _resolution_key(request_body.get("resolution")) + keyed_cost: Final = entry.get(f"output_cost_per_image_{resolution}") if resolution is not None else None + cost: Final = keyed_cost if isinstance(keyed_cost, (int, float)) else entry.get("output_cost_per_image") + return float(cost) if isinstance(cost, (int, float)) else None + + def cost_calculator( model: str, image_response: object, optional_params: Mapping[str, object] | None = None, + model_info: ModelInfo | None = None, ) -> float: """ fal.ai image generation cost calculator @@ -119,8 +164,14 @@ def cost_calculator( if not isinstance(image_response, ImageResponse): raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}") normalized_model: Final = model.removeprefix(f"{litellm.LlmProviders.FAL_AI.value}/") - params: Final[Mapping[str, object]] = optional_params or MappingProxyType({}) images: Final = tuple(image_response.data or ()) + deployment_prices: Final = deployment_pricing(model_info) + deployment_cost_per_image: Final = ( + None if deployment_prices is None else deployment_prices.get("output_cost_per_image") + ) + if deployment_cost_per_image is not None: + return deployment_cost_per_image * len(images) + params: Final[Mapping[str, object]] = optional_params or MappingProxyType({}) keyed_costs: Final = tuple( _keyed_cost_per_image( model=normalized_model, @@ -129,25 +180,28 @@ def cost_calculator( ) for image in images ) - if all(cost is not None for cost in keyed_costs): + if not any(cost is None for cost in keyed_costs): return sum(cost for cost in keyed_costs if cost is not None) - model_info: Final = litellm.get_model_info( + resolved_model_info: Final = resolve_image_model_info( model=normalized_model, custom_llm_provider=litellm.LlmProviders.FAL_AI.value, + model_info=deployment_prices, ) - raw_output_cost_per_image: Final = model_info.get("output_cost_per_image") + raw_output_cost_per_image: Final = resolved_model_info.get("output_cost_per_image") output_cost_per_image: Final = ( float(raw_output_cost_per_image) if isinstance(raw_output_cost_per_image, (int, float)) else 0.0 ) - raw_output_cost_per_pixel: Final = model_info.get("output_cost_per_pixel") + raw_output_cost_per_pixel: Final = resolved_model_info.get("output_cost_per_pixel") output_cost_per_pixel: Final = ( float(raw_output_cost_per_pixel) if isinstance(raw_output_cost_per_pixel, (int, float)) else None ) return sum( - _flat_cost_per_image( + keyed_cost + if keyed_cost is not None + else _flat_cost_per_image( image=image, output_cost_per_image=output_cost_per_image, output_cost_per_pixel=output_cost_per_pixel, ) - for image in images + for image, keyed_cost in zip(images, keyed_costs) ) diff --git a/litellm/llms/fal_ai/image_edit/__init__.py b/litellm/llms/fal_ai/image_edit/__init__.py index c2f0f311f8c..60775ef2c8d 100644 --- a/litellm/llms/fal_ai/image_edit/__init__.py +++ b/litellm/llms/fal_ai/image_edit/__init__.py @@ -1,3 +1,24 @@ +from typing import Final + +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig + +from .flux_lora_depth_transformation import FalAIFluxLoraDepthEditConfig from .transformation import FalAIImageEditConfig -__all__ = ("FalAIImageEditConfig",) +__all__ = ("FalAIFluxLoraDepthEditConfig", "FalAIImageEditConfig") + + +def get_fal_ai_image_edit_config(model: str) -> BaseImageEditConfig: + """ + Get the appropriate Fal AI image edit configuration based on the model. + + Args: + model: The Fal AI model name (e.g., "openai/gpt-image-2.5/flare/edit", "fal-ai/flux-lora-depth") + + Returns: + The appropriate configuration class for the specified model + """ + model_lower: Final = model.lower() + if "flux-lora-depth" in model_lower: + return FalAIFluxLoraDepthEditConfig() + return FalAIImageEditConfig() diff --git a/litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py b/litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py new file mode 100644 index 00000000000..fa469d638d2 --- /dev/null +++ b/litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py @@ -0,0 +1,75 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +from httpx._types import RequestFiles + +from litellm.secret_managers.main import get_secret_str +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import FileTypes + +from .transformation import DEFAULT_BASE_URL, FalAIImageEditConfig, to_data_url + +FLUX_LORA_DEPTH_ENDPOINT: Final[str] = "fal-ai/flux-lora-depth" +SUPPORTED_OPENAI_PARAMS: Final[tuple[str, ...]] = ("n", "size") +PARAM_TRANSLATION: Final[Mapping[str, str]] = MappingProxyType({"n": "num_images", "size": "image_size"}) + + +class FalAIFluxLoraDepthEditConfig(FalAIImageEditConfig): + """ + FLUX.1 [dev] depth LoRA edit endpoint served through Fal AI. + + Unlike the openai gpt-image ``/edit`` endpoints, this endpoint takes a single ``image_url`` + control image and has no ``/edit`` path suffix. + """ + + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: base class contract returns a list + return list(SUPPORTED_OPENAI_PARAMS) # mutable-ok: base class contract returns a list + + def map_openai_params( # mutable-ok: base class contract returns a dict + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict: # mutable-ok: base class contract returns a dict + return { # mutable-ok: base class contract returns a dict + PARAM_TRANSLATION.get(key, key): self._translate_value(key, value, model) + for key, value in image_edit_optional_params.items() + if value is not None and key in PARAM_TRANSLATION + } + + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict, # mutable-ok: base class contract + ) -> str: + base_url: Final = (api_base or get_secret_str("FAL_AI_API_BASE") or DEFAULT_BASE_URL).rstrip("/") + return f"{base_url}/{FLUX_LORA_DEPTH_ENDPOINT}" + + def transform_image_edit_request( + self, + model: str, + prompt: str | None, + image: FileTypes | None, + image_edit_optional_request_params: dict, # mutable-ok: base class contract + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: base class contract + ) -> tuple[dict, RequestFiles]: # mutable-ok: base class contract returns a dict + images: Final = tuple(img for img in (image if isinstance(image, list) else (image,)) if img is not None) + if not images: + raise ValueError("Fal AI image edit requires at least one input image") + if len(images) > 1: + raise ValueError(f"{FLUX_LORA_DEPTH_ENDPOINT} accepts exactly one control image") + provider_params: Final[Mapping[str, object]] = MappingProxyType( + { + key: value for key, value in image_edit_optional_request_params.items() if key != "mask" + } # mutable-ok: frozen by MappingProxyType + ) + request_body: Final[dict[str, object]] = { # mutable-ok: base class contract returns a dict + "prompt": prompt, + "image_url": to_data_url(next(iter(images))), + **provider_params, + } + return request_body, () diff --git a/litellm/llms/fal_ai/image_edit/transformation.py b/litellm/llms/fal_ai/image_edit/transformation.py index 70b5d0612f2..6e6a872839a 100644 --- a/litellm/llms/fal_ai/image_edit/transformation.py +++ b/litellm/llms/fal_ai/image_edit/transformation.py @@ -61,7 +61,7 @@ def _read_image_bytes(image: object) -> bytes: raise ValueError(f"Unsupported image type for Fal AI image edit: {type(image).__name__}") -def _to_data_url(image: object) -> str: +def to_data_url(image: object) -> str: if isinstance(image, str): return image image_bytes: Final = _read_image_bytes(image) @@ -143,7 +143,7 @@ class FalAIImageEditConfig(BaseImageEditConfig): raise ValueError("Fal AI image edit requires at least one input image") mask: Final = _first(image_edit_optional_request_params.get("mask")) mask_field: Final[Mapping[str, str]] = ( - MappingProxyType({"mask_url": _to_data_url(mask)}) if mask is not None else MappingProxyType({}) + MappingProxyType({"mask_url": to_data_url(mask)}) if mask is not None else MappingProxyType({}) ) provider_params: Final[Mapping[str, object]] = MappingProxyType( { @@ -152,7 +152,7 @@ class FalAIImageEditConfig(BaseImageEditConfig): ) request_body: Final[dict[str, object]] = { # mutable-ok: base class contract returns a dict "prompt": prompt, - "image_urls": tuple(_to_data_url(img) for img in images), + "image_urls": tuple(to_data_url(img) for img in images), **mask_field, **provider_params, } diff --git a/litellm/llms/fal_ai/image_generation/bria_transformation.py b/litellm/llms/fal_ai/image_generation/bria_transformation.py index c528550811a..53da48e62ca 100644 --- a/litellm/llms/fal_ai/image_generation/bria_transformation.py +++ b/litellm/llms/fal_ai/image_generation/bria_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -187,7 +186,7 @@ class FalAIBriaConfig(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py index 6b8558b8124..7c63e1077f1 100644 --- a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py +++ b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageResponse from .transformation import FalAIBaseConfig, fal_images_to_image_objects if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -194,7 +193,7 @@ class FalAIFluxProV11UltraConfig(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py b/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py index 04b4f426878..ad1852a622b 100644 --- a/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py +++ b/litellm/llms/fal_ai/image_generation/ideogram_v3_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -150,7 +149,7 @@ class FalAIIdeogramV3Config(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/imagen4_transformation.py b/litellm/llms/fal_ai/image_generation/imagen4_transformation.py index 8a6665b2585..3624b76a4a3 100644 --- a/litellm/llms/fal_ai/image_generation/imagen4_transformation.py +++ b/litellm/llms/fal_ai/image_generation/imagen4_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -182,7 +181,7 @@ class FalAIImagen4Config(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py b/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py index 4880dfec7e3..934ce420d53 100644 --- a/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py +++ b/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -172,7 +171,7 @@ class FalAIRecraftV3Config(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py index bc3a4d07282..79d8800773b 100644 --- a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py +++ b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py @@ -8,9 +8,8 @@ from litellm.types.utils import ImageObject, ImageResponse from .transformation import FalAIBaseConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -208,7 +207,7 @@ class FalAIStableDiffusionConfig(FalAIBaseConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/image_generation/transformation.py b/litellm/llms/fal_ai/image_generation/transformation.py index fd8e280da1c..8f081c7228d 100644 --- a/litellm/llms/fal_ai/image_generation/transformation.py +++ b/litellm/llms/fal_ai/image_generation/transformation.py @@ -16,9 +16,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -117,7 +116,7 @@ class FalAIBaseConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/fal_ai/videos/transformation.py b/litellm/llms/fal_ai/videos/transformation.py index 51082a6773b..e46199dd89f 100644 --- a/litellm/llms/fal_ai/videos/transformation.py +++ b/litellm/llms/fal_ai/videos/transformation.py @@ -104,7 +104,10 @@ def _profile_for_model(model: str) -> _ModelProfile: def _resolution_for_short_side(short_side: int, profile: _ModelProfile) -> str: - return next(resolution for threshold, resolution in profile.resolution_tiers if short_side <= threshold) + return next( + (resolution for threshold, resolution in profile.resolution_tiers if short_side <= threshold), + profile.resolution_tiers[-1][1], + ) def _model_path_from_request_url(raw_response: httpx.Response) -> str | None: @@ -227,6 +230,9 @@ def _response_string(response_data: Mapping[str, object], key: str, default: str return value if isinstance(value, str) else default +_RESULT_HEADERS_NOT_FORWARDED: Final[frozenset[str]] = frozenset({"host", "content-length", "transfer-encoding"}) + + def _result_request( raw_response: httpx.Response, response_data: Mapping[str, object], @@ -234,14 +240,12 @@ def _result_request( if _response_string(response_data, "status", "IN_QUEUE") != "COMPLETED": return None result_url: Final[str] = str(raw_response.request.url).removesuffix("/status") + encoding: Final[str] = raw_response.request.headers.encoding result_headers: Final[Mapping[str, str]] = MappingProxyType( { - key: value - for key, value in ( - ("Authorization", raw_response.request.headers.get("Authorization")), - ("Content-Type", raw_response.request.headers.get("Content-Type")), - ) - if value is not None + key.decode(encoding): value.decode(encoding) + for key, value in raw_response.request.headers.raw + if key.decode(encoding).lower() not in _RESULT_HEADERS_NOT_FORWARDED } ) return result_url, result_headers @@ -351,6 +355,8 @@ class FalAIVideoConfig(BaseVideoConfig): duration: Final[str | None] = _duration_value(seconds) if duration is None: raise ValueError("fal.ai seconds must be a numeric value") + if duration == "auto": + return MappingProxyType({}) if profile.integer_duration else MappingProxyType({"duration": duration}) return MappingProxyType({"duration": int(duration) if profile.integer_duration else duration}) def validate_environment( @@ -458,9 +464,10 @@ class FalAIVideoConfig(BaseVideoConfig): raw_response: httpx.Response, logging_obj: object, custom_llm_provider: str | None = None, + client: HTTPHandler | None = None, ) -> VideoObject: response_data: Final[Mapping[str, object]] = _response_data(raw_response) - result_error: Final[str | None] = self._fetch_result_error(raw_response, response_data) + result_error: Final[str | None] = self._fetch_result_error(raw_response, response_data, client) return _status_video_object( response_data=response_data, raw_response=raw_response, @@ -472,15 +479,20 @@ class FalAIVideoConfig(BaseVideoConfig): self, raw_response: httpx.Response, response_data: Mapping[str, object], + client: HTTPHandler | None, ) -> str | None: result_request: Final[tuple[str, Mapping[str, str]] | None] = _result_request(raw_response, response_data) if result_request is None: return None result_url, result_headers = result_request - result_response: Final[httpx.Response] = self._sync_client_factory().get( - url=result_url, - headers=result_headers, - ) + result_client: Final[HTTPHandler] = client if client is not None else self._sync_client_factory() + try: + result_response: Final[httpx.Response] = result_client.get( + url=result_url, + headers=dict(result_headers), # mutable-ok: HTTPHandler.get only accepts a dict + ) + except httpx.TransportError: + return None return _terminal_result_error(result_response) async def async_transform_video_status_retrieve_response( @@ -488,9 +500,10 @@ class FalAIVideoConfig(BaseVideoConfig): raw_response: httpx.Response, logging_obj: object, custom_llm_provider: str | None = None, + client: AsyncHTTPHandler | None = None, ) -> VideoObject: response_data: Final[Mapping[str, object]] = _response_data(raw_response) - result_error: Final[str | None] = await self._fetch_result_error_async(raw_response, response_data) + result_error: Final[str | None] = await self._fetch_result_error_async(raw_response, response_data, client) return _status_video_object( response_data=response_data, raw_response=raw_response, @@ -502,15 +515,20 @@ class FalAIVideoConfig(BaseVideoConfig): self, raw_response: httpx.Response, response_data: Mapping[str, object], + client: AsyncHTTPHandler | None, ) -> str | None: result_request: Final[tuple[str, Mapping[str, str]] | None] = _result_request(raw_response, response_data) if result_request is None: return None result_url, result_headers = result_request - result_response: Final[httpx.Response] = await self._async_client_factory().get( - url=result_url, - headers=result_headers, - ) + result_client: Final[AsyncHTTPHandler] = client if client is not None else self._async_client_factory() + try: + result_response: Final[httpx.Response] = await result_client.get( + url=result_url, + headers=dict(result_headers), # mutable-ok: AsyncHTTPHandler.get only accepts a dict + ) + except httpx.TransportError: + return None return _terminal_result_error(result_response) @staticmethod diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 28ebb39a303..196022b3558 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -46,7 +46,7 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer def _map_reasoning_effort(value: object) -> object: @@ -708,7 +708,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/gemini/chat/transformation.py b/litellm/llms/gemini/chat/transformation.py index 42c9ef13730..285350aecba 100644 --- a/litellm/llms/gemini/chat/transformation.py +++ b/litellm/llms/gemini/chat/transformation.py @@ -102,7 +102,7 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig): "include_server_side_tool_invocations", "service_tier", ] - if supports_reasoning(model, custom_llm_provider="gemini"): + if supports_reasoning(model, custom_llm_provider="gemini") or self._is_gemini_3_or_newer(model): supported_params.append("reasoning_effort") supported_params.append("thinking") if self.is_model_gemini_audio_model(model): diff --git a/litellm/llms/gemini/image_edit/cost_calculator.py b/litellm/llms/gemini/image_edit/cost_calculator.py index 956edb849a0..321fbbaeb37 100644 --- a/litellm/llms/gemini/image_edit/cost_calculator.py +++ b/litellm/llms/gemini/image_edit/cost_calculator.py @@ -7,11 +7,13 @@ from typing import Any from litellm.llms.gemini.image_generation.cost_calculator import ( cost_calculator as image_generation_cost_calculator, ) +from litellm.types.utils import ModelInfo def cost_calculator( model: str, image_response: Any, + model_info: ModelInfo | None = None, ) -> float: """ Gemini image edit cost calculator. @@ -22,4 +24,5 @@ def cost_calculator( return image_generation_cost_calculator( model=model, image_response=image_response, + model_info=model_info, ) diff --git a/litellm/llms/gemini/image_generation/cost_calculator.py b/litellm/llms/gemini/image_generation/cost_calculator.py index ea0e77e1b81..e3232f8fb7b 100644 --- a/litellm/llms/gemini/image_generation/cost_calculator.py +++ b/litellm/llms/gemini/image_generation/cost_calculator.py @@ -4,24 +4,26 @@ Google AI Image Generation Cost Calculator from typing import Any, Final -import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_image_response_cost_from_usage, calculate_image_response_web_search_cost, + resolve_image_model_info, ) -from litellm.types.utils import ImageResponse +from litellm.types.utils import ImageResponse, ModelInfo def cost_calculator( model: str, image_response: Any, + model_info: ModelInfo | None = None, ) -> float: """ Google AI Image Generation Cost Calculator """ - _model_info: Final = litellm.get_model_info( + _model_info: Final = resolve_image_model_info( model=model, custom_llm_provider="gemini", + model_info=model_info, ) if not isinstance(image_response, ImageResponse): @@ -37,6 +39,7 @@ def cost_calculator( model=model, image_response=image_response, custom_llm_provider="gemini", + model_info=_model_info, ) if token_based_cost is not None: return token_based_cost + web_search_cost diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index d009fe4cd72..bb8d7455031 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -24,9 +24,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -173,7 +172,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py index a44717eb659..8a3894cf7fb 100644 --- a/litellm/llms/gemini/videos/transformation.py +++ b/litellm/llms/gemini/videos/transformation.py @@ -26,6 +26,7 @@ from litellm.types.videos.utils import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.llms.custom_httpx.http_handler import HTTPHandler from ...base_llm.chat.transformation import BaseLLMException as _BaseLLMException @@ -386,6 +387,7 @@ class GeminiVideoConfig(BaseVideoConfig): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, custom_llm_provider: str | None = None, + client: "HTTPHandler | None" = None, ) -> VideoObject: """ Transform the Veo operation status response. diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index d9250ea8836..c047dc0c881 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -26,9 +26,8 @@ from ..authenticator import get_access_token from ..file_handler import upload_file_sync if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -416,7 +415,7 @@ class GigaChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: tiktoken.Encoding | None, + encoding: Tokenizer | None, api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 41a2df17c6f..1da7ad0a7b5 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -27,7 +27,7 @@ from litellm.types.utils import ModelResponse, ModelResponseStream, ServerToolUs from ...openai_like.chat.transformation import OpenAILikeChatConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer GROQ_COMPOUND_MODELS: Final = frozenset({"compound", "compound-mini"}) @@ -286,7 +286,7 @@ class GroqChatConfig(OpenAILikeChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/huggingface/embedding/handler.py b/litellm/llms/huggingface/embedding/handler.py index 57d1357ee46..60917c68221 100644 --- a/litellm/llms/huggingface/embedding/handler.py +++ b/litellm/llms/huggingface/embedding/handler.py @@ -1,6 +1,5 @@ import json import os -from collections.abc import Sequence from typing import Final, Literal, Protocol, get_args import httpx @@ -32,7 +31,7 @@ hf_tasks_embeddings: Final = ( class _SupportsTokenEncode(Protocol): """Token encoder handle. Only ``encode`` is ever called on it here.""" - def encode(self, text: str, *, disallowed_special: tuple[str, ...]) -> Sequence[int]: ... + def encode(self, text: str) -> list[int]: ... def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> str | None: @@ -214,7 +213,7 @@ class HuggingFaceEmbedding(BaseLLM): model_response.model = model input_tokens = 0 for text in input: - input_tokens += len(encoding.encode(text, disallowed_special=())) + input_tokens += len(encoding.encode_ordinary(text)) setattr( model_response, diff --git a/litellm/llms/huggingface/embedding/transformation.py b/litellm/llms/huggingface/embedding/transformation.py index 33b0e21e326..3fdd4abda73 100644 --- a/litellm/llms/huggingface/embedding/transformation.py +++ b/litellm/llms/huggingface/embedding/transformation.py @@ -25,9 +25,8 @@ from litellm.utils import token_counter from ..common_utils import HuggingFaceError, hf_task_list, hf_tasks, output_parser if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -479,7 +478,7 @@ class HuggingFaceEmbeddingConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/langflow/chat/transformation.py b/litellm/llms/langflow/chat/transformation.py index 17ae7017cf6..a53887b36af 100644 --- a/litellm/llms/langflow/chat/transformation.py +++ b/litellm/llms/langflow/chat/transformation.py @@ -14,9 +14,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import CustomStreamWrapper @@ -225,7 +224,7 @@ class LangFlowConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/langgraph/chat/transformation.py b/litellm/llms/langgraph/chat/transformation.py index 84d79e6bd31..c9388ee472f 100644 --- a/litellm/llms/langgraph/chat/transformation.py +++ b/litellm/llms/langgraph/chat/transformation.py @@ -23,9 +23,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import CustomStreamWrapper @@ -415,7 +414,7 @@ class LangGraphConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index c01ad2a0edd..341e8dd2e12 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -19,7 +19,7 @@ from litellm.types.utils import ModelResponse from ...openai_like.chat.transformation import OpenAILikeChatConfig if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class LemonadeChatConfig(OpenAILikeChatConfig): @@ -231,7 +231,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index f77e828b59a..33b567e9710 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -32,7 +32,7 @@ from litellm.types.utils import ModelResponse, ModelResponseStream from litellm.utils import convert_to_model_response_object, supports_reasoning if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: str) -> str: @@ -580,7 +580,7 @@ class MistralConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/nlp_cloud/chat/transformation.py b/litellm/llms/nlp_cloud/chat/transformation.py index 17c547618d3..2ff48894619 100644 --- a/litellm/llms/nlp_cloud/chat/transformation.py +++ b/litellm/llms/nlp_cloud/chat/transformation.py @@ -14,9 +14,8 @@ from litellm.utils import ModelResponse, Usage from ..common_utils import NLPCloudError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -175,7 +174,7 @@ class NLPCloudConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 8e4e41b4ac1..ecff823a18d 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -65,9 +65,8 @@ from litellm.types.utils import ( from litellm.utils import supports_reasoning if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -603,7 +602,7 @@ class OCIChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index bcde8a041a6..cb3080e6534 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -31,9 +31,8 @@ from litellm.types.utils import ModelResponse, ModelResponseStream from ..common_utils import OllamaError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -321,7 +320,7 @@ class OllamaChatConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 3eb2c833094..0fc1cd926b8 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -35,9 +35,8 @@ from litellm.types.utils import ( from ..common_utils import OllamaError, OllamaModelInfo, _convert_image if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -252,7 +251,7 @@ class OllamaConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/oobabooga/chat/transformation.py b/litellm/llms/oobabooga/chat/transformation.py index 43d627102b6..05383a35389 100644 --- a/litellm/llms/oobabooga/chat/transformation.py +++ b/litellm/llms/oobabooga/chat/transformation.py @@ -11,9 +11,8 @@ from litellm.types.utils import ModelResponse, Usage from ..common_utils import OobaboogaError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -39,7 +38,7 @@ class OobaboogaConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 9dbcf0cc089..b63684db782 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -60,9 +60,8 @@ from litellm.utils import convert_to_model_response_object from ..common_utils import OpenAIError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.types.llms.openai import ChatCompletionToolParam @@ -671,7 +670,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openai/image_generation/cost_calculator.py b/litellm/llms/openai/image_generation/cost_calculator.py index 938a0a57f2a..350fdbaee2d 100644 --- a/litellm/llms/openai/image_generation/cost_calculator.py +++ b/litellm/llms/openai/image_generation/cost_calculator.py @@ -9,27 +9,37 @@ from typing import Final from litellm import verbose_logger from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_image_response_cost_from_usage, + flat_image_cost, generic_cost_per_token, + resolve_image_model_info, ) -from litellm.types.utils import ImageResponse, Usage +from litellm.types.utils import ImageResponse, ModelInfo, Usage def cost_calculator( model: str, image_response: ImageResponse, custom_llm_provider: str | None = None, + model_info: ModelInfo | None = None, ) -> float: """Calculate cost for OpenAI gpt-image models (token-based pricing).""" + provider: Final = custom_llm_provider or "openai" + price_table: Final = ( + None + if model_info is None + else resolve_image_model_info(model=model, custom_llm_provider=provider, model_info=model_info) + ) + usage: Final = getattr(image_response, "usage", None) if usage is None: verbose_logger.debug("No usage data available for %s, cannot calculate token-based cost", model) - return 0.0 - - provider: Final = custom_llm_provider or "openai" + return flat_image_cost(price_table, image_response) # A chat Usage with an explicit output breakdown: cost via generic_cost_per_token. if isinstance(usage, Usage) and usage.completion_tokens_details is not None: - prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider=provider) + prompt_cost, completion_cost = generic_cost_per_token( + model=model, usage=usage, custom_llm_provider=provider, model_info=price_table + ) return prompt_cost + completion_cost # ImageUsage / ResponseAPIUsage: reuse the shared helper (same path as @@ -38,7 +48,7 @@ def cost_calculator( # does not itemize output and splitting text/image when it does. if getattr(usage, "input_tokens", None) is not None: token_based_cost: Final = calculate_image_response_cost_from_usage( - model=model, image_response=image_response, custom_llm_provider=provider + model=model, image_response=image_response, custom_llm_provider=provider, model_info=price_table ) if token_based_cost is not None: return token_based_cost @@ -46,7 +56,9 @@ def cost_calculator( # Fallback: a Usage with no output breakdown that the image helper can't read — # cost via generic_cost_per_token (text rate) instead of returning 0.0. if isinstance(usage, Usage): - prompt_cost, completion_cost = generic_cost_per_token(model=model, usage=usage, custom_llm_provider=provider) + prompt_cost, completion_cost = generic_cost_per_token( + model=model, usage=usage, custom_llm_provider=provider, model_info=price_table + ) return prompt_cost + completion_cost - return 0.0 + return flat_image_cost(price_table, image_response) diff --git a/litellm/llms/openai/image_generation/dall_e_2_transformation.py b/litellm/llms/openai/image_generation/dall_e_2_transformation.py index 74936cf1895..ffc6f1d5fe9 100644 --- a/litellm/llms/openai/image_generation/dall_e_2_transformation.py +++ b/litellm/llms/openai/image_generation/dall_e_2_transformation.py @@ -10,9 +10,10 @@ from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: - import tiktoken from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class DallE2ImageGenerationConfig(BaseImageGenerationConfig): """ @@ -52,7 +53,7 @@ class DallE2ImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/openai/image_generation/dall_e_3_transformation.py b/litellm/llms/openai/image_generation/dall_e_3_transformation.py index 5c561d011a9..90b7eaedf2f 100644 --- a/litellm/llms/openai/image_generation/dall_e_3_transformation.py +++ b/litellm/llms/openai/image_generation/dall_e_3_transformation.py @@ -10,9 +10,10 @@ from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: - import tiktoken from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class DallE3ImageGenerationConfig(BaseImageGenerationConfig): """ @@ -52,7 +53,7 @@ class DallE3ImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/openai/image_generation/gpt_transformation.py b/litellm/llms/openai/image_generation/gpt_transformation.py index 8dc4d8953ea..c3a826616ed 100644 --- a/litellm/llms/openai/image_generation/gpt_transformation.py +++ b/litellm/llms/openai/image_generation/gpt_transformation.py @@ -10,9 +10,10 @@ from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object if TYPE_CHECKING: - import tiktoken from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class GPTImageGenerationConfig(BaseImageGenerationConfig): """ @@ -61,7 +62,7 @@ class GPTImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/openai/image_variations/transformation.py b/litellm/llms/openai/image_variations/transformation.py index afd2909b697..73b44d5ea6a 100644 --- a/litellm/llms/openai/image_variations/transformation.py +++ b/litellm/llms/openai/image_variations/transformation.py @@ -12,7 +12,7 @@ from ...base_llm.image_variations.transformation import BaseImageVariationConfig from ..common_utils import OpenAIError if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class OpenAIImageVariationConfig(BaseImageVariationConfig): @@ -53,7 +53,7 @@ class OpenAIImageVariationConfig(BaseImageVariationConfig): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: return model_response @@ -68,7 +68,7 @@ class OpenAIImageVariationConfig(BaseImageVariationConfig): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: return model_response diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 869ad387c5a..7ac0d988074 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -6,9 +6,10 @@ from typing import TYPE_CHECKING, Final, Literal, Optional, cast import httpx if TYPE_CHECKING: - import tiktoken from aiohttp import ClientSession + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + import openai from openai import AsyncOpenAI, OpenAI from openai._base_client import make_request_options @@ -277,7 +278,7 @@ class OpenAIConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index e3ecbac1a53..bdc3a6c7908 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -12,6 +12,7 @@ from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.types.realtime import RealtimeQueryParams from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from ....litellm_core_utils.realtime_errors import close_after_upstream_handshake_refusal from ....litellm_core_utils.realtime_streaming import ( RealtimeEventNormalizer, RealTimeStreaming, @@ -175,8 +176,8 @@ class OpenAIRealtime(OpenAIChatCompletion): ) await realtime_streaming.bidirectional_forward() - except websockets.exceptions.InvalidStatusCode as e: - await websocket.close(code=e.status_code, reason=_redact_string(str(e))) + except websockets.exceptions.InvalidStatus as e: + await close_after_upstream_handshake_refusal(websocket, e.response.status_code) except Exception as e: try: await websocket.close(code=1011, reason=_redact_string(f"Internal server error: {e}")) diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index 9a4b030993f..cfd4ca24ae7 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -28,6 +28,7 @@ from litellm.types.videos.utils import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.llms.custom_httpx.http_handler import HTTPHandler from ...base_llm.chat.transformation import BaseLLMException as _BaseLLMException @@ -437,6 +438,7 @@ class OpenAIVideoConfig(BaseVideoConfig): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, custom_llm_provider: str | None = None, + client: "HTTPHandler | None" = None, ) -> VideoObject: """ Transform the OpenAI video retrieve response. diff --git a/litellm/llms/openai_like/chat/transformation.py b/litellm/llms/openai_like/chat/transformation.py index 030710c8b2d..e5d6cbb7e5e 100644 --- a/litellm/llms/openai_like/chat/transformation.py +++ b/litellm/llms/openai_like/chat/transformation.py @@ -13,9 +13,8 @@ from litellm.types.utils import ModelResponse from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -131,7 +130,7 @@ class OpenAILikeChatConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index 77a902149d9..08d43c169f4 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -23,9 +23,8 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig from ..common_utils import OpenRouterException if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class CacheControlSupportedModels(str, Enum): @@ -182,7 +181,7 @@ class OpenrouterConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/openrouter/image_generation/transformation.py b/litellm/llms/openrouter/image_generation/transformation.py index 6bbda324336..67d90d027ec 100644 --- a/litellm/llms/openrouter/image_generation/transformation.py +++ b/litellm/llms/openrouter/image_generation/transformation.py @@ -50,9 +50,8 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer else: LiteLLMLoggingObj = Any @@ -319,7 +318,7 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/perplexity/chat/transformation.py b/litellm/llms/perplexity/chat/transformation.py index 354f7692fd5..dca2f9857b8 100644 --- a/litellm/llms/perplexity/chat/transformation.py +++ b/litellm/llms/perplexity/chat/transformation.py @@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues, ChatCompletionAnnotation from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class PerplexityChatConfig(OpenAIGPTConfig): @@ -75,7 +75,7 @@ class PerplexityChatConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/petals/completion/transformation.py b/litellm/llms/petals/completion/transformation.py index 3e0de14a7b2..ee20c2b12d5 100644 --- a/litellm/llms/petals/completion/transformation.py +++ b/litellm/llms/petals/completion/transformation.py @@ -14,7 +14,7 @@ from litellm.types.utils import ModelResponse from ..common_utils import PetalsError if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class PetalsConfig(BaseConfig): @@ -112,7 +112,7 @@ class PetalsConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/predibase/chat/transformation.py b/litellm/llms/predibase/chat/transformation.py index 2a63c489395..69924396a1e 100644 --- a/litellm/llms/predibase/chat/transformation.py +++ b/litellm/llms/predibase/chat/transformation.py @@ -18,9 +18,8 @@ from litellm.types.utils import Choices, Message, ModelResponse, Usage from ..common_utils import PredibaseError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -150,7 +149,7 @@ class PredibaseConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/recraft/cost_calculator.py b/litellm/llms/recraft/cost_calculator.py index 2866d8e1d96..221d4a35913 100644 --- a/litellm/llms/recraft/cost_calculator.py +++ b/litellm/llms/recraft/cost_calculator.py @@ -1,19 +1,22 @@ from typing import Any, Final import litellm -from litellm.types.utils import ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.types.utils import ImageResponse, ModelInfo def cost_calculator( model: str, image_response: Any, + model_info: ModelInfo | None = None, ) -> float: """ Recraft image generation cost calculator """ - _model_info: Final = litellm.get_model_info( + _model_info: Final = resolve_image_model_info( model=model, custom_llm_provider=litellm.LlmProviders.RECRAFT.value, + model_info=model_info, ) output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 diff --git a/litellm/llms/recraft/image_generation/transformation.py b/litellm/llms/recraft/image_generation/transformation.py index 3a04e0a62b4..f65bf1e7292 100644 --- a/litellm/llms/recraft/image_generation/transformation.py +++ b/litellm/llms/recraft/image_generation/transformation.py @@ -14,9 +14,8 @@ from litellm.types.llms.recraft import RecraftImageGenerationRequestParams from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -122,7 +121,7 @@ class RecraftImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/replicate/chat/transformation.py b/litellm/llms/replicate/chat/transformation.py index 769160c6ced..f7e09b7bec0 100644 --- a/litellm/llms/replicate/chat/transformation.py +++ b/litellm/llms/replicate/chat/transformation.py @@ -19,9 +19,8 @@ from litellm.utils import token_counter from ..common_utils import ReplicateError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LoggingClass = LiteLLMLoggingObj else: @@ -237,7 +236,7 @@ class ReplicateConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/runwayml/cost_calculator.py b/litellm/llms/runwayml/cost_calculator.py index fdd4b904b60..07f7d564ac2 100644 --- a/litellm/llms/runwayml/cost_calculator.py +++ b/litellm/llms/runwayml/cost_calculator.py @@ -1,12 +1,14 @@ from typing import Any, Final import litellm -from litellm.types.utils import ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.types.utils import ImageResponse, ModelInfo def cost_calculator( model: str, image_response: Any, + model_info: ModelInfo | None = None, ) -> float: """ RunwayML image generation cost calculator. @@ -14,9 +16,10 @@ def cost_calculator( RunwayML charges per image generated, not per pixel. Pricing is stored in model_prices_and_context_window.json with output_cost_per_image. """ - _model_info: Final = litellm.get_model_info( + _model_info: Final = resolve_image_model_info( model=model, custom_llm_provider=litellm.LlmProviders.RUNWAYML.value, + model_info=model_info, ) output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 diff --git a/litellm/llms/runwayml/image_generation/transformation.py b/litellm/llms/runwayml/image_generation/transformation.py index 5913709c8a0..e5e988328d8 100644 --- a/litellm/llms/runwayml/image_generation/transformation.py +++ b/litellm/llms/runwayml/image_generation/transformation.py @@ -22,9 +22,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -308,7 +307,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: @@ -383,7 +382,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index c7696a1cb29..1abdece44f2 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -616,6 +616,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, custom_llm_provider: str | None = None, + client: HTTPHandler | None = None, ) -> VideoObject: """ Transform the RunwayML video status retrieve response. diff --git a/litellm/llms/sagemaker/completion/transformation.py b/litellm/llms/sagemaker/completion/transformation.py index 576018f0046..e1bc496a82d 100644 --- a/litellm/llms/sagemaker/completion/transformation.py +++ b/litellm/llms/sagemaker/completion/transformation.py @@ -24,9 +24,8 @@ from litellm.utils import token_counter from ..common_utils import SagemakerError if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -198,7 +197,7 @@ class SagemakerConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index d64d7a57281..4c73ccacc16 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -15,9 +15,8 @@ from litellm.types.utils import ModelResponse from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -57,7 +56,7 @@ def validate_dict(data: dict, model) -> dict: return model(**data).model_dump(by_alias=True, exclude_unset=True) -def _messages_to_sap_template(messages: list[dict[str, str]]) -> list: +def _messages_to_sap_template(messages: list[AllMessageValues]) -> list: template: Final = [] for message in messages: if message["role"] == "user": @@ -311,7 +310,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): def transform_request( self, model: str, - messages: list[dict[str, str]], + messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, headers: dict, @@ -383,7 +382,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/stability/image_generation/transformation.py b/litellm/llms/stability/image_generation/transformation.py index cf3576a9404..656ffe395c8 100644 --- a/litellm/llms/stability/image_generation/transformation.py +++ b/litellm/llms/stability/image_generation/transformation.py @@ -26,9 +26,8 @@ from litellm.types.llms.stability import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -207,7 +206,7 @@ class StabilityImageGenerationConfig(BaseImageGenerationConfig): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/topaz/image_variations/transformation.py b/litellm/llms/topaz/image_variations/transformation.py index f4753c8ba17..94f60d29cb9 100644 --- a/litellm/llms/topaz/image_variations/transformation.py +++ b/litellm/llms/topaz/image_variations/transformation.py @@ -23,7 +23,7 @@ from ...base_llm.image_variations.transformation import BaseImageVariationConfig from ..common_utils import TopazException, TopazModelInfo if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig): @@ -139,7 +139,7 @@ class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: image_content: Final = await raw_response.read() @@ -158,7 +158,7 @@ class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig): image: FileTypes, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, ) -> ImageResponse: image_content: Final = raw_response.content diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index 3c868b3a96f..b37bbd78f2b 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -29,7 +29,7 @@ from litellm.types.utils import ( from ..common_utils import TritonError if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class TritonConfig(BaseConfig): @@ -95,7 +95,7 @@ class TritonConfig(BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: @@ -215,7 +215,7 @@ class TritonGenerateConfig(TritonConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: @@ -280,7 +280,7 @@ class TritonInferConfig(TritonConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/vertex_ai/agent_engine/transformation.py b/litellm/llms/vertex_ai/agent_engine/transformation.py index c5ca9f38144..b37bf473731 100644 --- a/litellm/llms/vertex_ai/agent_engine/transformation.py +++ b/litellm/llms/vertex_ai/agent_engine/transformation.py @@ -29,9 +29,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.utils import CustomStreamWrapper @@ -285,7 +284,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 46f1b948026..204b1ddd5ea 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -2,6 +2,7 @@ ## httpx client for vertex ai calls ## Initial implementation - covers gemini + image gen calls import json +import re import time from collections.abc import Callable, Mapping, Sequence from copy import deepcopy @@ -283,20 +284,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): @staticmethod def _is_gemini_3_or_newer(model: str) -> bool: """ - Check if the model is Gemini 3 Pro or newer. - - Gemini 3 models include: - - gemini-3-pro-preview - - gemini-3-flash - - gemini-3-flash-preview (Gemini 3 Flash) - - gemini-3.1-pro-preview, gemini-3.1-flash, gemini-3.1-flash-lite-preview - - gemini-3.5-flash - - Any future Gemini 3.x models + Check if the model is Gemini 3 or newer. """ - # Check for Gemini 3 models - if "gemini-3" in model: - return True - return False + model_name = model.split("/")[-1].lower() + if not model_name: + return False + # Pre-Gemini 3 models: gemini-1.x, gemini-2.x, gemini-pro, gemini-flash, gemini-exp + if re.match(r"^gemini-(?:[12](?:\.\d+)?|exp|(?:pro|flash)(?!-(?:lite-)?latest$))(?:-|$)", model_name): + return False + return True @staticmethod def _forward_gemini_function_call_id(model: str) -> bool: @@ -347,9 +343,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if self._supports_penalty_parameters(model): supported_params.extend(["frequency_penalty", "presence_penalty"]) - if supports_reasoning(model): + if supports_reasoning(model) or self._is_gemini_3_or_newer(model): supported_params.append("reasoning_effort") supported_params.append("thinking") + return supported_params def map_tool_choice_values(self, model: str, tool_choice: str | dict) -> ToolConfig | None: @@ -871,8 +868,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): @staticmethod def _supports_minimal_thinking_level(model: str) -> bool: lowered: Final = model.lower() - is_gemini3flash: Final = "gemini-3" in lowered and "flash" in lowered - return is_gemini3flash and not is_explicitly_disabled_factory( + is_gemini3_or_newer_flash: Final = VertexGeminiConfig._is_gemini_3_or_newer(model) and "flash" in lowered + return is_gemini3_or_newer_flash and not is_explicitly_disabled_factory( model=model, custom_llm_provider=None, key="supports_minimal_reasoning_effort" ) @@ -890,9 +887,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): Returns: GeminiThinkingConfig with thinkingLevel and includeThoughts """ - is_gemini3flash: Final = model and ("flash" in model.lower() and "gemini-3" in model.lower()) supports_minimal: Final = bool(model) and VertexGeminiConfig._supports_minimal_thinking_level(model) - is_gemini31pro: Final = model and ("gemini-3.1-pro-preview" in model.lower()) if reasoning_effort == "minimal": if supports_minimal: return {"thinkingLevel": "minimal", "includeThoughts": True} @@ -901,10 +896,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif reasoning_effort == "low": return {"thinkingLevel": "low", "includeThoughts": True} elif reasoning_effort == "medium": - if is_gemini31pro or is_gemini3flash: - return {"thinkingLevel": "medium", "includeThoughts": True} - else: - return {"thinkingLevel": "high", "includeThoughts": True} + return {"thinkingLevel": "medium", "includeThoughts": True} elif reasoning_effort == "high": return {"thinkingLevel": "high", "includeThoughts": True} elif reasoning_effort in ("disable", "none"): diff --git a/litellm/llms/vertex_ai/image_generation/cost_calculator.py b/litellm/llms/vertex_ai/image_generation/cost_calculator.py index f3117e65681..6db4f02ee5a 100644 --- a/litellm/llms/vertex_ai/image_generation/cost_calculator.py +++ b/litellm/llms/vertex_ai/image_generation/cost_calculator.py @@ -4,24 +4,26 @@ Vertex AI Image Generation Cost Calculator from typing import Final -import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import ( calculate_image_response_cost_from_usage, calculate_image_response_web_search_cost, + resolve_image_model_info, ) -from litellm.types.utils import ImageResponse +from litellm.types.utils import ImageResponse, ModelInfo def cost_calculator( model: str, image_response: ImageResponse, + model_info: ModelInfo | None = None, ) -> float: """ Vertex AI Image Generation Cost Calculator """ - _model_info: Final = litellm.get_model_info( + _model_info: Final = resolve_image_model_info( model=model, custom_llm_provider="vertex_ai", + model_info=model_info, ) web_search_cost: Final = calculate_image_response_web_search_cost( @@ -34,6 +36,7 @@ def cost_calculator( model=model, image_response=image_response, custom_llm_provider="vertex_ai", + model_info=_model_info, ) if token_based_cost is not None: return token_based_cost + web_search_cost diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py index b2c52c53580..17ccf16837b 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -24,9 +24,8 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -284,7 +283,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py index 8faf7b0d484..ae6e08611ae 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py @@ -20,9 +20,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -214,7 +213,7 @@ class VertexAIImagenImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): request_data: dict, optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 508f68b3eca..2fab6f438f6 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -14,7 +14,7 @@ from ....anthropic.chat.transformation import AnthropicConfig from .output_params_utils import sanitize_vertex_anthropic_output_params if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class VertexAIError(Exception): @@ -197,7 +197,7 @@ class VertexAIAnthropicConfig(AnthropicConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py index 89a5b8a570e..f2d2c0896d2 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py @@ -21,7 +21,7 @@ from litellm.types.utils import ( from ...common_utils import VertexAIError if TYPE_CHECKING: - import tiktoken + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer class VertexAILlama3Config(OpenAIGPTConfig): @@ -112,7 +112,7 @@ class VertexAILlama3Config(OpenAIGPTConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py index 67b01c2dc43..2ae8b4cd188 100644 --- a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py +++ b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py @@ -24,9 +24,8 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.base_llm.base_model_iterator import MockResponseIterator @@ -276,7 +275,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): litellm_params: dict, client: HTTPHandler | httpx.Client | None = None, timeout: float | httpx.Timeout | None = None, - encoding: "tiktoken.Encoding | None" = None, + encoding: "Tokenizer | None" = None, ): """Synchronous completion request""" from litellm.utils import convert_to_model_response_object @@ -366,7 +365,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): litellm_params: dict, client: AsyncHTTPHandler | httpx.AsyncClient | None = None, timeout: float | httpx.Timeout | None = None, - encoding: "tiktoken.Encoding | None" = None, + encoding: "Tokenizer | None" = None, ): """Asynchronous completion request""" from litellm.utils import convert_to_model_response_object diff --git a/litellm/llms/vertex_ai/videos/transformation.py b/litellm/llms/vertex_ai/videos/transformation.py index dc9caa13224..6c29059f0a9 100644 --- a/litellm/llms/vertex_ai/videos/transformation.py +++ b/litellm/llms/vertex_ai/videos/transformation.py @@ -36,6 +36,7 @@ if TYPE_CHECKING: from litellm.llms.base_llm.chat.transformation import ( BaseLLMException as _BaseLLMException, ) + from litellm.llms.custom_httpx.http_handler import HTTPHandler LiteLLMLoggingObj = _LiteLLMLoggingObj BaseLLMException = _BaseLLMException @@ -491,6 +492,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, custom_llm_provider: str | None = None, + client: "HTTPHandler | None" = None, ) -> VideoObject: """ Transform the Veo operation status response. diff --git a/litellm/llms/watsonx/completion/transformation.py b/litellm/llms/watsonx/completion/transformation.py index 2be007336b4..2fec8485cf9 100644 --- a/litellm/llms/watsonx/completion/transformation.py +++ b/litellm/llms/watsonx/completion/transformation.py @@ -21,9 +21,8 @@ from ..common_utils import ( ) if TYPE_CHECKING: - import tiktoken - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -280,7 +279,7 @@ class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig): messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: "tiktoken.Encoding | None", + encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: diff --git a/litellm/main.py b/litellm/main.py index 6704358e3ea..7f4b34d28a0 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -37,7 +37,6 @@ if TYPE_CHECKING: import dotenv import httpx import openai -import tiktoken from pydantic import BaseModel from typing_extensions import overload @@ -79,8 +78,9 @@ from litellm.litellm_core_utils.chat_completion_agentic_loop import ( from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_litellm_params import ( - FORWARDED_KWARGS_KEYS, + AWS_CREDENTIAL_KWARGS_KEYS, OPTIONAL_KWARGS_KEYS, + PROVIDER_AFFINITY_HEADER_KWARG_KEY, ) from litellm.litellm_core_utils.get_provider_specific_headers import ( ProviderSpecificHeaderUtils, @@ -97,9 +97,11 @@ from litellm.litellm_core_utils.mock_functions import ( from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, ) +from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, ) +from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer from litellm.llms.azure_ai.common_utils import ( azure_ai_supports_native_responses, foundry_chat_rejects_function_tools_while_reasoning, @@ -3594,6 +3596,37 @@ def _complete_edenai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResu return response +def _complete_fal_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: + if ctx.stream: + raise litellm.FalAIError( + status_code=400, + message="fal_ai chat completions do not support streaming", + ) + api_base: Final = litellm.FalAIChatConfig.get_api_base(ctx.api_base) + api_key: Final = litellm.FalAIChatConfig.get_api_key(ctx.api_key or litellm.api_key) + response: Final = base_llm_http_handler.completion( + model=ctx.model, + messages=ctx.messages, + api_base=api_base, + custom_llm_provider="fal_ai", + model_response=ctx.model_response, + encoding=_get_encoding(), + logging_obj=ctx.logging, + optional_params=ctx.optional_params, + timeout=ctx.timeout, + litellm_params=ctx.litellm_params, + shared_session=ctx.shared_session, + acompletion=ctx.acompletion, + stream=ctx.stream, + api_key=api_key, + headers=ctx.headers or litellm.headers, + client=_dispatch_client_http(ctx), + provider_config=ctx.provider_config, + ) + ctx.logging.post_call(input=ctx.messages, api_key=api_key, original_response=response) + return response + + def _complete_vertex_ai_beta( ctx: _CompletionDispatchContext, ) -> _CompletionDispatchResult: @@ -5613,8 +5646,30 @@ def completion( gigachat_scope=kwargs.get("gigachat_scope"), gigachat_auth_url=kwargs.get("gigachat_auth_url"), gigachat_access_token=kwargs.get("gigachat_access_token"), - **{key: kwargs[key] for key in FORWARDED_KWARGS_KEYS if key in kwargs}, + **{ + key: kwargs[key] + for key in (*AWS_CREDENTIAL_KWARGS_KEYS, PROVIDER_AFFINITY_HEADER_KWARG_KEY) + if key in kwargs + }, ) + if litellm_params.get("provider_affinity_header") is not None: + try: + headers = add_provider_affinity_header( + headers=headers or litellm.headers or MappingProxyType({}), + litellm_params=MappingProxyType( + { + "provider_affinity_header": litellm_params["provider_affinity_header"], + "litellm_session_id": kwargs.get("litellm_session_id"), + "session_id": kwargs.get("session_id"), + "metadata": metadata, + "litellm_metadata": kwargs.get("litellm_metadata"), + } + ), + ) + except ValueError as affinity_error: + raise litellm.BadRequestError( + message=str(affinity_error), model=model, llm_provider=custom_llm_provider + ) from affinity_error cast(LiteLLMLoggingObj, logging).update_environment_variables( model=model, user=user, @@ -5799,6 +5854,8 @@ def completion( response = _complete_hosted_vllm(_dispatch_ctx) elif custom_llm_provider == "edenai": response = _complete_edenai(_dispatch_ctx) # rebind-ok: dispatch chain binds response per branch + elif custom_llm_provider == "fal_ai": + response = _complete_fal_ai(_dispatch_ctx) # rebind-ok: dispatch chain binds response per branch elif ( # A known OpenAI model name only decides the route when nothing else # resolved a provider. get_llm_provider() already maps these names to @@ -7456,7 +7513,9 @@ def text_completion( if isinstance(prompt, list): import concurrent.futures - tokenizer: Final = tiktoken.encoding_for_model("text-davinci-003") + from litellm.rust_bridge.tokenizer import get_encoding + + tokenizer: Final = get_encoding("p50k_base") ## if it's a 2d list - each element in the list is a text_completion() request if len(prompt) > 0 and isinstance(prompt[0], list): responses: Final = [None for x in prompt] # init responses @@ -9226,7 +9285,7 @@ async def acount_tokens( except Exception as e: verbose_logger.debug("Provider token counting failed for model=%s, falling back to local: %s", model, e) - # Fallback to local tiktoken-based token counting + # Fallback to local token counting fallback_messages = messages or [] if system and fallback_messages: fallback_messages = [{"role": "system", "content": system}] + fallback_messages @@ -9245,16 +9304,16 @@ async def acount_tokens( # Cache for encoding to avoid repeated __getattr__ calls -_encoding_cache: tiktoken.Encoding | None = None +_encoding_cache: Tokenizer | None = None -def _load_module_encoding() -> tiktoken.Encoding: +def _load_module_encoding() -> Tokenizer: import sys return sys.modules[__name__].encoding -def _get_encoding() -> tiktoken.Encoding: +def _get_encoding() -> Tokenizer: """Get encoding, loading it lazily if needed.""" global _encoding_cache if _encoding_cache is None: @@ -9263,18 +9322,15 @@ def _get_encoding() -> tiktoken.Encoding: return _encoding_cache -def _load_default_encoding() -> tiktoken.Encoding: +def _load_default_encoding() -> Tokenizer: from litellm._lazy_imports import _get_default_encoding return _get_default_encoding() -def __getattr__(name: str) -> tiktoken.Encoding: +def __getattr__(name: str) -> Tokenizer: """Lazy import handler for main module""" if name == "encoding": - # Use _get_default_encoding which properly sets TIKTOKEN_CACHE_DIR - # before loading tiktoken, ensuring the local cache is used - # instead of downloading from the internet _encoding: Final = _load_default_encoding() # Cache it in the module's __dict__ for subsequent accesses import sys diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index c75f6564d1b..a0c791a136c 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -4,7 +4,7 @@ from types import MappingProxyType from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable from litellm.llms.anthropic.experimental_pass_through.messages import handler as main -from litellm.rust_bridge.catalog import Context, Delivery, Route +from litellm.rust_bridge.catalog import Delivery, Route, RouteContext from litellm.rust_bridge.dispatch import PublicDispatch, call_hook from litellm.rust_bridge.messages.entrypoints import ( NATIVE_AMESSAGES, @@ -71,8 +71,8 @@ def _public_request( ) -def _context(request: LiteLLMMessagesRequest) -> Context: - return Context( +def _context(request: LiteLLMMessagesRequest) -> RouteContext: + return RouteContext( Route.MESSAGES, provider=request.custom_llm_provider, model=request.model, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 80bb2bf9bd2..224129fbd25 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -53,13 +53,6 @@ "mode": "image_generation", "output_cost_per_image": 0.04 }, - "1024-x-1024/dall-e-2": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 1.9e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "1024-x-1024/max-steps/stability.stable-diffusion-xl-v1": { "litellm_provider": "bedrock", "max_input_tokens": 77, @@ -67,13 +60,6 @@ "mode": "image_generation", "output_cost_per_image": 0.08 }, - "256-x-256/dall-e-2": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 2.4414e-07, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "512-x-512/50-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, @@ -81,13 +67,6 @@ "mode": "image_generation", "output_cost_per_image": 0.018 }, - "512-x-512/dall-e-2": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 6.86e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "512-x-512/max-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, @@ -578,17 +557,6 @@ "supports_vision": true, "supports_tool_choice": true }, - "amazon.nova-sonic-v1:0": { - "deprecation_date": "2026-09-14", - "input_cost_per_audio_token": 3.4e-06, - "input_cost_per_token": 6e-08, - "litellm_provider": "bedrock", - "mode": "realtime", - "output_cost_per_audio_token": 1.36e-05, - "output_cost_per_token": 2.4e-07, - "supports_audio_input": true, - "supports_audio_output": true - }, "amazon.nova-2-sonic-v1:0": { "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 3.3e-07, @@ -972,23 +940,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 - }, "anthropic.claude-3-opus-20240229-v1:0": { "input_cost_per_token": 1.5e-05, "litellm_provider": "bedrock", @@ -1004,23 +955,6 @@ "cache_read_input_token_cost": 1.5e-06, "cache_creation_input_token_cost": 1.875e-05 }, - "anthropic.claude-3-sonnet-20240229-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, "anthropic.claude-instant-v1": { "input_cost_per_token": 8e-07, "litellm_provider": "bedrock", @@ -1839,6 +1773,46 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "thinking_always_on": true, + "supports_forced_tool_use": false + }, "global.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1877,6 +1851,46 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "global.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "thinking_always_on": true, + "supports_forced_tool_use": false + }, "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1915,6 +1929,46 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "us.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_1hr": 8.8e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 4.4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "thinking_always_on": true, + "supports_forced_tool_use": false + }, "eu.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1952,6 +2006,46 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512 }, + "eu.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_1hr": 8.8e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 4.4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "au.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1989,6 +2083,46 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512 }, + "au.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_1hr": 8.8e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 4.4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "jp.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -2026,6 +2160,46 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512 }, + "jp.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_1hr": 8.8e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 4.4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -2982,60 +3156,6 @@ "supports_vision": true, "supports_tool_choice": true }, - "apac.anthropic.claude-3-5-sonnet-20240620-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, - "apac.anthropic.claude-3-5-sonnet-20241022-v2:0": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "apac.anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 - }, "apac.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, "cache_read_input_token_cost": 1.1e-07, @@ -3063,23 +3183,6 @@ "input_cost_per_token_batches": 5.5e-07, "output_cost_per_token_batches": 2.75e-06 }, - "apac.anthropic.claude-3-sonnet-20240229-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, "apac.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, @@ -3424,6 +3527,41 @@ "supports_max_reasoning_effort": true, "prompt_cache_min_tokens": 512 }, + "azure_ai/claude-opus-5-5": { + "supports_mid_conversation_system": true, + "input_cost_per_token": 4e-06, + "output_cost_per_token": 2e-05, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 + }, "azure_ai/claude-opus-4-8": { "deprecation_date": "2027-09-01", "supports_mid_conversation_system": true, @@ -3457,29 +3595,6 @@ "supports_max_reasoning_effort": true, "prompt_cache_min_tokens": 1024 }, - "azure_ai/claude-opus-4-1": { - "deprecation_date": "2026-08-05", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "azure_ai", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, "azure_ai/claude-sonnet-4-5": { "deprecation_date": "2026-10-19", "cache_creation_input_token_cost": 3.75e-06, @@ -4428,7 +4543,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 6.875e-08 }, "azure/eu/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-08, @@ -4467,11 +4583,13 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.375e-08 }, "azure/eu/gpt-5.1": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_batches": 6.875e-08, "cache_read_input_token_cost_priority": 2.75e-07, "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", @@ -4511,43 +4629,6 @@ "output_cost_per_token_priority": 2.2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/eu/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.375e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.375e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/eu/gpt-5.1-codex": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.375e-07, @@ -4646,7 +4727,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 2.75e-09 }, "azure/eu/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, @@ -4684,21 +4766,6 @@ "supports_prompt_caching": true, "supports_vision": false }, - "azure/eu/o1-preview-2024-09-12": { - "cache_read_input_token_cost": 8.25e-06, - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_vision": false - }, "azure/eu/o3-mini-2025-01-31": { "cache_read_input_token_cost": 6.05e-07, "deprecation_date": "2026-11-19", @@ -4814,6 +4881,7 @@ "azure/global/gpt-5.1": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -4853,43 +4921,6 @@ "output_cost_per_token_priority": 2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/global/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/global/gpt-5.1-codex": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, @@ -4965,19 +4996,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "azure/gpt-3.5-turbo-0125": { - "deprecation_date": "2025-03-31", - "input_cost_per_token": 5e-07, - "litellm_provider": "azure", - "max_input_tokens": 16384, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true - }, "azure/gpt-3.5-turbo-instruct-0914": { "input_cost_per_token": 1.5e-06, "litellm_provider": "azure_text", @@ -4997,32 +5015,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "azure/gpt-35-turbo-0125": { - "deprecation_date": "2025-05-31", - "input_cost_per_token": 5e-07, - "litellm_provider": "azure", - "max_input_tokens": 16384, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true - }, - "azure/gpt-35-turbo-1106": { - "deprecation_date": "2025-03-31", - "input_cost_per_token": 1e-06, - "litellm_provider": "azure", - "max_input_tokens": 16384, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true - }, "azure/gpt-35-turbo-16k": { "input_cost_per_token": 3e-06, "litellm_provider": "azure", @@ -5790,40 +5782,6 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "azure/gpt-realtime-2": { - "cache_read_input_audio_token_cost": 4e-07, - "cache_read_input_token_cost": 4e-07, - "deprecation_date": "2026-08-31", - "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image_token": 5e-06, - "input_cost_per_token": 4e-06, - "litellm_provider": "azure", - "max_input_tokens": 32000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 6.4e-05, - "output_cost_per_token": 2.4e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", - "supported_endpoints": [ - "/v1/realtime" - ], - "supported_modalities": [ - "text", - "image", - "audio" - ], - "supported_output_modalities": [ - "text", - "audio" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "azure/gpt-realtime-2.1": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, @@ -6118,46 +6076,8 @@ "input_cost_per_token_batches": 6.25e-07, "output_cost_per_token_batches": 5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_minimal_reasoning_effort": true - }, - "azure/gpt-5.1-chat-2025-11-13": { - "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_priority": 2.5e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "input_cost_per_token_priority": 2.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "output_cost_per_token_priority": 2e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_native_streaming": true, - "supports_parallel_function_calling": false, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": false, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_batches": 6.25e-08 }, "azure/gpt-5.1-codex-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, @@ -6232,6 +6152,7 @@ "azure/gpt-5": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, "input_cost_per_token_batches": 6.25e-07, @@ -6305,74 +6226,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true - }, - "azure/gpt-5-chat": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "azure/gpt-5-chat-latest": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 6.25e-08 }, "azure/gpt-5-codex": { "cache_read_input_token_cost": 1.25e-07, @@ -6409,6 +6264,7 @@ "azure/gpt-5-mini": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, @@ -6482,11 +6338,13 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.25e-08 }, "azure/gpt-5-nano": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_batches": 2.5e-09, "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, "litellm_provider": "azure", @@ -6554,7 +6412,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 2.5e-09 }, "azure/gpt-5-pro": { "deprecation_date": "2027-04-07", @@ -6591,6 +6450,7 @@ "azure/gpt-5.1": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -6630,43 +6490,6 @@ "output_cost_per_token_priority": 2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/gpt-5.1-codex": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, @@ -6766,6 +6589,7 @@ "azure/gpt-5.2": { "deprecation_date": "2027-06-08", "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_batches": 8.75e-08, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, "input_cost_per_token_batches": 8.75e-07, @@ -6841,79 +6665,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true - }, - "azure/gpt-5.2-chat": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "azure/gpt-5.2-chat-2025-12-11": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-05-13", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 8.75e-08 }, "azure/gpt-5.2-codex": { "cache_read_input_token_cost": 1.75e-07, @@ -6947,42 +6700,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "azure/gpt-5.3-chat": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "azure/gpt-5.3-codex": { "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, @@ -7100,9 +6817,11 @@ "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07, "cache_read_input_token_cost_priority": 5e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, + "cache_read_input_token_cost_batches": 1.3e-07, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_priority": 5e-06, @@ -7140,9 +6859,11 @@ "supports_vision": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, "input_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_flex": 7.5e-06, @@ -7154,6 +6875,8 @@ "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.75e-07, "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, + "cache_read_input_token_cost_batches": 1.43e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, "input_cost_per_token_priority": 5.5e-06, @@ -7189,8 +6912,10 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "input_cost_per_token_above_272k_tokens": 5.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.75e-06, "input_cost_per_token_batches": 1.375e-06, "output_cost_per_token_above_272k_tokens": 2.475e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.2375e-05, "output_cost_per_token_batches": 8.25e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, @@ -7200,6 +6925,8 @@ "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.75e-07, "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, + "cache_read_input_token_cost_batches": 1.43e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, "input_cost_per_token_priority": 5.5e-06, @@ -7235,8 +6962,10 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "input_cost_per_token_above_272k_tokens": 5.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.75e-06, "input_cost_per_token_batches": 1.375e-06, "output_cost_per_token_above_272k_tokens": 2.475e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.2375e-05, "output_cost_per_token_batches": 8.25e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, @@ -7294,7 +7023,11 @@ "output_cost_per_token_flex": 7.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, + "cache_read_input_token_cost_batches": 1.3e-07, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05 }, "azure/us/gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.75e-07, @@ -7340,7 +7073,11 @@ "output_cost_per_token_batches": 8.25e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, + "cache_read_input_token_cost_batches": 1.43e-07, + "input_cost_per_token_above_272k_tokens_batches": 2.75e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.2375e-05 }, "azure/eu/gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.75e-07, @@ -7386,7 +7123,11 @@ "output_cost_per_token_batches": 8.25e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, + "cache_read_input_token_cost_batches": 1.43e-07, + "input_cost_per_token_above_272k_tokens_batches": 2.75e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.2375e-05 }, "azure/gpt-5.4-pro": { "deprecation_date": "2027-09-07", @@ -7394,6 +7135,7 @@ "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "input_cost_per_token_above_272k_tokens_flex": 3e-05, "input_cost_per_token_batches": 1.5e-05, "input_cost_per_token_flex": 1.5e-05, @@ -7404,6 +7146,7 @@ "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "output_cost_per_token_above_272k_tokens_flex": 0.000135, "output_cost_per_token_batches": 9e-05, "output_cost_per_token_flex": 9e-05, @@ -7482,7 +7225,9 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135 }, "azure/gpt-5.6": { "cache_creation_input_token_cost": 6.25e-06, @@ -8625,11 +8370,14 @@ "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 2e-06, + "cache_read_input_token_cost_batches": 2.5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_above_272k_tokens_flex": 5e-06, "input_cost_per_token_priority": 1.25e-05, "input_cost_per_token_above_272k_tokens_priority": 2e-05, @@ -8642,6 +8390,7 @@ "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "output_cost_per_token_above_272k_tokens_priority": 9e-05, "output_cost_per_token_batches": 1.5e-05, @@ -8682,9 +8431,12 @@ "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, + "cache_read_input_token_cost_batches": 2.75e-07, "cache_read_input_token_cost_priority": 1.375e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, + "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, "input_cost_per_token_batches": 2.75e-06, "input_cost_per_token_priority": 1.375e-05, "output_cost_per_token": 3.3e-05, @@ -8695,6 +8447,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, "output_cost_per_token_batches": 1.65e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -8733,9 +8486,12 @@ "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, + "cache_read_input_token_cost_batches": 2.75e-07, "cache_read_input_token_cost_priority": 1.375e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, + "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, "input_cost_per_token_batches": 2.75e-06, "input_cost_per_token_priority": 1.375e-05, "output_cost_per_token": 3.3e-05, @@ -8746,6 +8502,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, "output_cost_per_token_batches": 1.65e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -8783,8 +8540,10 @@ "azure/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 2e-06, + "cache_read_input_token_cost_batches": 2.5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, @@ -8828,9 +8587,11 @@ "supports_vision": true, "supports_web_search": true, "deprecation_date": "2027-10-26", + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_above_272k_tokens_flex": 5e-06, "input_cost_per_token_batches": 2.5e-06, "input_cost_per_token_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_batches": 1.5e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, @@ -8889,11 +8650,17 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05 }, "azure/us/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, + "cache_read_input_token_cost_batches": 2.75e-07, "cache_read_input_token_cost_priority": 1.375e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, @@ -8935,7 +8702,9 @@ "supports_vision": true, "supports_web_search": true, "deprecation_date": "2027-10-26", + "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, "output_cost_per_token_batches": 1.65e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, @@ -8988,11 +8757,17 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, + "cache_read_input_token_cost_batches": 2.75e-07, + "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.475e-05 }, "azure/eu/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, + "cache_read_input_token_cost_batches": 2.75e-07, "cache_read_input_token_cost_priority": 1.375e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, @@ -9034,7 +8809,9 @@ "supports_vision": true, "supports_web_search": true, "deprecation_date": "2027-10-26", + "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, "output_cost_per_token_batches": 1.65e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, @@ -9087,7 +8864,11 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, + "cache_read_input_token_cost_batches": 2.75e-07, + "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.475e-05 }, "azure/gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -9176,6 +8957,7 @@ "azure/gpt-5.4-mini": { "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -9273,11 +9055,13 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_priority": 9e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "cache_read_input_token_cost_batches": 3.75e-08 }, "azure/gpt-5.4-nano": { "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2e-08, + "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "input_cost_per_token": 2e-07, "litellm_provider": "azure", @@ -9369,7 +9153,8 @@ "output_cost_per_token_batches": 6.25e-07, "output_cost_per_token_flex": 6.25e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "cache_read_input_token_cost_batches": 1e-08 }, "azure/gpt-image-1": { "cache_read_input_token_cost": 1.25e-06, @@ -9772,39 +9557,6 @@ "supports_reasoning": true, "supports_vision": false }, - "azure/o1-preview": { - "cache_read_input_token_cost": 7.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_vision": false - }, - "azure/o1-preview-2024-09-12": { - "cache_read_input_token_cost": 7.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_vision": false - }, "azure/o3": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5e-07, @@ -10416,7 +10168,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 6.875e-08 }, "azure/us/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-08, @@ -10455,7 +10208,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.375e-08 }, "azure/us/gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5.5e-09, @@ -10491,11 +10245,13 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 2.75e-09 }, "azure/us/gpt-5.1": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_batches": 6.875e-08, "cache_read_input_token_cost_priority": 2.75e-07, "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", @@ -10535,43 +10291,6 @@ "output_cost_per_token_priority": 2.2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/us/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.375e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.375e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/us/gpt-5.1-codex": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.375e-07, @@ -10672,21 +10391,6 @@ "supports_prompt_caching": true, "supports_vision": false }, - "azure/us/o1-preview-2024-09-12": { - "cache_read_input_token_cost": 8.25e-06, - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_vision": false - }, "azure/us/o3-2025-04-16": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, @@ -11188,18 +10892,6 @@ "/v1/images/edits" ] }, - "azure_ai/MAI-Image-2e": { - "deprecation_date": "2026-08-15", - "input_cost_per_token": 5e-06, - "litellm_provider": "azure_ai", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "output_cost_per_image_token": 1.95e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/images/generations" - ] - }, "azure_ai/MAI-Thinking-1": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 2e-06, @@ -11224,34 +10916,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "azure_ai/Llama-3.2-11B-Vision-Instruct": { - "deprecation_date": "2026-06-13", - "input_cost_per_token": 3.7e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 3.7e-07, - "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-11b-vision-instruct-offer?tab=Overview", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "azure_ai/Llama-3.2-90B-Vision-Instruct": { - "deprecation_date": "2026-06-13", - "input_cost_per_token": 2.04e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 2.04e-06, - "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-90b-vision-instruct-offer?tab=Overview", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "azure_ai/Llama-3.3-70B-Instruct": { "input_cost_per_token": 7.1e-07, "litellm_provider": "azure_ai", @@ -11300,18 +10964,6 @@ "output_cost_per_token": 3.7e-07, "supports_tool_choice": true }, - "azure_ai/Meta-Llama-3.1-405B-Instruct": { - "deprecation_date": "2026-06-13", - "input_cost_per_token": 5.33e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 1.6e-05, - "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-405b-instruct-offer?tab=PlansAndPrice", - "supports_tool_choice": true - }, "azure_ai/Meta-Llama-3.1-70B-Instruct": { "input_cost_per_token": 2.68e-06, "litellm_provider": "azure_ai", @@ -11323,18 +10975,6 @@ "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-70b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, - "azure_ai/Meta-Llama-3.1-8B-Instruct": { - "deprecation_date": "2026-06-13", - "input_cost_per_token": 3e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 6.1e-07, - "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-8b-instruct-offer?tab=PlansAndPrice", - "supports_tool_choice": true - }, "azure_ai/Phi-3-medium-128k-instruct": { "input_cost_per_token": 1.7e-07, "litellm_provider": "azure_ai", @@ -11505,16 +11145,6 @@ "supports_tool_choice": true, "supports_reasoning": true }, - "azure_ai/mistral-document-ai-2505": { - "deprecation_date": "2026-07-20", - "litellm_provider": "azure_ai", - "ocr_cost_per_page": 0.003, - "mode": "ocr", - "supported_endpoints": [ - "/v1/ocr" - ], - "source": "https://devblogs.microsoft.com/foundry/whats-new-in-azure-ai-foundry-august-2025/#mistral-document-ai-(ocr)-%E2%80%94-serverless-in-foundry" - }, "azure_ai/mistral-document-ai-2512": { "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.003, @@ -11615,17 +11245,6 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, - "azure_ai/cohere-rerank-v3.5": { - "deprecation_date": "2026-05-14", - "input_cost_per_query": 0.002, - "input_cost_per_token": 0.0, - "litellm_provider": "azure_ai", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "rerank", - "output_cost_per_token": 0.0 - }, "azure_ai/cohere-rerank-v4.0-pro": { "input_cost_per_query": 0.0025, "input_cost_per_token": 0.0, @@ -11678,19 +11297,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "azure_ai/deepseek-r1": { - "deprecation_date": "2026-08-13", - "input_cost_per_token": 1.35e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 5.4e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_reasoning": true, - "supports_tool_choice": true - }, "azure_ai/deepseek-v3": { "input_cost_per_token": 1.14e-06, "litellm_provider": "azure_ai", @@ -11702,33 +11308,6 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true }, - "azure_ai/deepseek-v3-0324": { - "deprecation_date": "2026-07-13", - "input_cost_per_token": 1.14e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4.56e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_tool_choice": true - }, - "azure_ai/deepseek-v3.1": { - "deprecation_date": "2026-07-13", - "input_cost_per_token": 1.23e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 4.94e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "azure_ai/deepseek-v4-pro": { "deprecation_date": "2028-02-20", "input_cost_per_token": 1.74e-06, @@ -11795,68 +11374,6 @@ ], "supports_embedding_image_input": true }, - "azure_ai/global/grok-3": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 3e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true - }, - "azure_ai/global/grok-3-mini": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.27e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true - }, - "azure_ai/grok-3": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 3e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true - }, - "azure_ai/grok-3-mini": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.27e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true - }, "azure_ai/grok-4": { "input_cost_per_token": 3e-06, "litellm_provider": "azure_ai", @@ -11950,36 +11467,6 @@ "supports_vision": true, "supports_web_search": true }, - "azure_ai/grok-4-fast-non-reasoning": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 2e-07, - "output_cost_per_token": 5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "azure_ai/grok-4-fast-reasoning": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 2e-07, - "output_cost_per_token": 5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_web_search": true - }, "azure_ai/grok-4-1-fast-non-reasoning": { "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, @@ -12934,6 +12421,23 @@ "supports_tool_choice": true, "output_cost_per_token": 1.86e-06 }, + "bedrock/eu-west-2/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 2.3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.01e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, "litellm_provider": "bedrock", @@ -13486,40 +12990,6 @@ "mode": "chat", "output_cost_per_token": 1.5e-06 }, - "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3.6e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3.6e-07, - "cache_creation_input_token_cost": 4.5e-06 - }, - "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 3e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-08, - "cache_creation_input_token_cost": 3.75e-07 - }, "bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, @@ -13697,61 +13167,6 @@ "mode": "chat", "output_cost_per_token": 1.5e-06 }, - "bedrock/us-gov-west-1/anthropic.claude-3-7-sonnet-20250219-v1:0": { - "cache_creation_input_token_cost": 4.5e-06, - "cache_read_input_token_cost": 3.6e-07, - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3.6e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3.6e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3.6e-07, - "cache_creation_input_token_cost": 4.5e-06 - }, - "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 3e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-08, - "cache_creation_input_token_cost": 3.75e-07 - }, "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, @@ -14189,34 +13604,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "cerebras/zai-glm-4.6": { - "deprecation_date": "2026-01-20", - "input_cost_per_token": 2.25e-06, - "litellm_provider": "cerebras", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.75e-06, - "source": "https://www.cerebras.ai/pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, - "cerebras/zai-glm-4.7": { - "deprecation_date": "2026-08-17", - "input_cost_per_token": 2.25e-06, - "litellm_provider": "cerebras", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.75e-06, - "source": "https://www.cerebras.ai/pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "cerebras/qwen-3.8-27b": { "input_cost_per_token": 9.9e-07, "litellm_provider": "cerebras", @@ -14242,23 +13629,6 @@ "mode": "chat", "output_cost_per_token": 5e-07 }, - "chatgpt-4o-latest": { - "deprecation_date": "2026-02-17", - "input_cost_per_token": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "gpt-4o-transcribe-diarize": { "input_cost_per_audio_token": 2.5e-06, "input_cost_per_token": 2.5e-06, @@ -14321,131 +13691,6 @@ "prompt_cache_min_tokens": 4096, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, - "claude-3-7-sonnet-20250219": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-02-19", - "input_cost_per_token": 3e-06, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, - "claude-3-haiku-20240307": { - "cache_creation_input_token_cost": 3e-07, - "cache_creation_input_token_cost_above_1hr": 5e-07, - "cache_read_input_token_cost": 3e-08, - "deprecation_date": "2026-04-20", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "claude-3-opus-20240229": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "deprecation_date": "2026-01-05", - "input_cost_per_token": 1.5e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "claude-4-opus-20250514": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_read_input_token_cost": 1.5e-06, - "deprecation_date": "2026-06-15", - "input_cost_per_token": 1.5e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "claude-4-sonnet-20250514": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost": 3e-07, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "deprecation_date": "2026-06-15", - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "litellm_provider": "anthropic", - "max_input_tokens": 1000000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "prompt_cache_min_tokens": 1024 - }, "claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -14516,6 +13761,7 @@ "source": "https://docs.anthropic.com/en/docs/about-claude/pricing" }, "claude-sonnet-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -14555,6 +13801,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-sonnet-4-6": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -14622,92 +13869,6 @@ "input_cost_per_token_batches": 1.5e-06, "output_cost_per_token_batches": 7.5e-06 }, - "claude-opus-4-1": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024, - "deprecation_date": "2026-08-05" - }, - "claude-opus-4-1-20250805": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "deprecation_date": "2026-08-05", - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "claude-opus-4-20250514": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "deprecation_date": "2026-06-15", - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, "claude-opus-4-5-20251101": { "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -14770,6 +13931,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-6": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14808,6 +13970,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-6-20260205": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14845,6 +14008,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-7": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14884,6 +14048,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-7-20260416": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14922,6 +14087,7 @@ "prompt_cache_min_tokens": 2048 }, "claude-fable-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -14961,6 +14127,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-fable-5-1": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 2.5e-07, @@ -15000,7 +14167,51 @@ "supports_native_structured_output": true, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, + "claude-opus-5-5": { + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "fast": 2.0 + }, + "supports_output_config": true, + "supports_speed": true, + "supports_fast_mode": true, + "prompt_cache_min_tokens": 512, + "source": "https://platform.claude.com/docs/en/about-claude/pricing" + }, "claude-opus-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -15043,6 +14254,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-8": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -15084,38 +14296,6 @@ "prompt_cache_min_tokens": 1024, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, - "claude-sonnet-4-20250514": { - "deprecation_date": "2026-06-15", - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "litellm_provider": "anthropic", - "max_input_tokens": 1000000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, "litellm_provider": "cloudflare", @@ -15468,36 +14648,6 @@ "supports_assistant_prefill": true, "supports_tool_choice": true }, - "codex-mini-latest": { - "cache_read_input_token_cost": 3.75e-07, - "deprecation_date": "2026-02-12", - "input_cost_per_token": 1.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 6e-06, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "cohere.command-light-text-v14": { "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", @@ -15508,28 +14658,6 @@ "output_cost_per_token": 6e-07, "supports_tool_choice": true }, - "cohere.command-r-plus-v1:0": { - "deprecation_date": "2026-08-19", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_tool_choice": true - }, - "cohere.command-r-v1:0": { - "deprecation_date": "2026-08-19", - "input_cost_per_token": 5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_tool_choice": true - }, "cohere.command-text-v14": { "input_cost_per_token": 1.5e-06, "litellm_provider": "bedrock", @@ -15619,16 +14747,6 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, - "command": { - "input_cost_per_token": 1e-06, - "litellm_provider": "cohere", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "completion", - "output_cost_per_token": 2e-06, - "deprecation_date": "2025-09-15" - }, "command-a-03-2025": { "input_cost_per_token": 2.5e-06, "litellm_provider": "cohere_chat", @@ -15655,17 +14773,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "command-light": { - "input_cost_per_token": 3e-07, - "litellm_provider": "cohere_chat", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_tool_choice": true, - "deprecation_date": "2025-09-15" - }, "command-nightly": { "input_cost_per_token": 1e-06, "litellm_provider": "cohere", @@ -15675,18 +14782,6 @@ "mode": "completion", "output_cost_per_token": 2e-06 }, - "command-r": { - "input_cost_per_token": 1.5e-07, - "litellm_provider": "cohere_chat", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_function_calling": true, - "supports_tool_choice": true, - "deprecation_date": "2025-09-15" - }, "command-r-08-2024": { "input_cost_per_token": 1.5e-07, "litellm_provider": "cohere_chat", @@ -15698,18 +14793,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "command-r-plus": { - "input_cost_per_token": 2.5e-06, - "litellm_provider": "cohere_chat", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "deprecation_date": "2025-09-15" - }, "command-r-plus-08-2024": { "input_cost_per_token": 2.5e-06, "litellm_provider": "cohere_chat", @@ -15762,26 +14845,6 @@ "supports_vision": true, "source": "https://platform.openai.com/docs/models/computer-use-preview" }, - "dall-e-2": { - "deprecation_date": "2026-05-12", - "input_cost_per_image": 0.02, - "litellm_provider": "openai", - "mode": "image_generation", - "supported_endpoints": [ - "/v1/images/generations", - "/v1/images/edits", - "/v1/images/variations" - ] - }, - "dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_image": 0.04, - "litellm_provider": "openai", - "mode": "image_generation", - "supported_endpoints": [ - "/v1/images/generations" - ] - }, "deepseek-chat": { "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 2.8e-07, @@ -18778,30 +17841,6 @@ "output_vector_size": 1024, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, - "databricks/databricks-claude-3-7-sonnet": { - "cache_creation_input_token_cost": 3.74997e-06, - "cache_read_input_token_cost": 3.0002e-07, - "deprecation_date": "2026-04-12", - "input_cost_per_token": 2.9999900000000002e-06, - "input_dbu_cost_per_token": 4.2857e-05, - "litellm_provider": "databricks", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, - "output_dbu_cost_per_token": 0.000214286, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_anthropic_thinking_payload": true, - "supports_tool_choice": true - }, "databricks/databricks-claude-fable-5": { "cache_creation_input_token_cost": 1.250004e-05, "cache_read_input_token_cost": 1.00002e-06, @@ -19764,44 +18803,6 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, - "databricks/databricks-gpt-5-1-codex-max": { - "cache_creation_input_token_cost": 1.24999e-06, - "cache_read_input_token_cost": 1.2502e-07, - "deprecation_date": "2026-07-16", - "input_cost_per_token": 1.24999e-06, - "input_dbu_cost_per_token": 1.7857e-05, - "litellm_provider": "databricks", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 9.999990000000002e-06, - "output_dbu_cost_per_token": 0.000142857, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true - }, - "databricks/databricks-gpt-5-1-codex-mini": { - "cache_creation_input_token_cost": 2.4997e-07, - "cache_read_input_token_cost": 2.499e-08, - "deprecation_date": "2026-07-16", - "input_cost_per_token": 2.4997e-07, - "input_dbu_cost_per_token": 3.571e-06, - "litellm_provider": "databricks", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.99997e-06, - "output_dbu_cost_per_token": 2.8571e-05, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true - }, "databricks/databricks-gpt-5-2": { "cache_creation_input_token_cost": 1.75e-06, "cache_read_input_token_cost": 1.75e-07, @@ -19822,25 +18823,6 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, - "databricks/databricks-gpt-5-2-codex": { - "cache_creation_input_token_cost": 1.75e-06, - "cache_read_input_token_cost": 1.75e-07, - "deprecation_date": "2026-07-16", - "input_cost_per_token": 1.75e-06, - "input_dbu_cost_per_token": 2.5e-05, - "litellm_provider": "databricks", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_dbu_cost_per_token": 0.0002, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true - }, "databricks/databricks-gpt-5-3-codex": { "cache_creation_input_token_cost": 1.75e-06, "cache_read_input_token_cost": 1.75e-07, @@ -20267,25 +19249,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "databricks/databricks-llama-2-70b-chat": { - "cache_creation_input_token_cost": 5.0001e-07, - "cache_read_input_token_cost": 5.0001e-07, - "deprecation_date": "2024-10-30", - "input_cost_per_token": 5.0001e-07, - "input_dbu_cost_per_token": 7.143e-06, - "litellm_provider": "databricks", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.5000300000000002e-06, - "output_dbu_cost_per_token": 2.1429e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, "databricks/databricks-llama-4-maverick": { "cache_creation_input_token_cost": 5.0001e-07, "cache_read_input_token_cost": 5.0001e-07, @@ -20304,25 +19267,6 @@ "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true }, - "databricks/databricks-meta-llama-3-1-405b-instruct": { - "cache_creation_input_token_cost": 5.00003e-06, - "cache_read_input_token_cost": 5.00003e-06, - "deprecation_date": "2026-02-15", - "input_cost_per_token": 5.00003e-06, - "input_dbu_cost_per_token": 7.1429e-05, - "litellm_provider": "databricks", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, - "output_dbu_cost_per_token": 0.000214286, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, "databricks/databricks-meta-llama-3-1-8b-instruct": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, @@ -20358,82 +19302,6 @@ "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true }, - "databricks/databricks-meta-llama-3-70b-instruct": { - "cache_creation_input_token_cost": 1.00002e-06, - "cache_read_input_token_cost": 1.00002e-06, - "deprecation_date": "2024-07-23", - "input_cost_per_token": 1.00002e-06, - "input_dbu_cost_per_token": 1.4286e-05, - "litellm_provider": "databricks", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 2.9999900000000002e-06, - "output_dbu_cost_per_token": 4.2857e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, - "databricks/databricks-mixtral-8x7b-instruct": { - "cache_creation_input_token_cost": 5.0001e-07, - "cache_read_input_token_cost": 5.0001e-07, - "deprecation_date": "2025-04-30", - "input_cost_per_token": 5.0001e-07, - "input_dbu_cost_per_token": 7.143e-06, - "litellm_provider": "databricks", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.00002e-06, - "output_dbu_cost_per_token": 1.4286e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, - "databricks/databricks-mpt-30b-instruct": { - "cache_creation_input_token_cost": 1.00002e-06, - "cache_read_input_token_cost": 1.00002e-06, - "deprecation_date": "2024-08-30", - "input_cost_per_token": 1.00002e-06, - "input_dbu_cost_per_token": 1.4286e-05, - "litellm_provider": "databricks", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.00002e-06, - "output_dbu_cost_per_token": 1.4286e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, - "databricks/databricks-mpt-7b-instruct": { - "cache_creation_input_token_cost": 5.0001e-07, - "cache_read_input_token_cost": 5.0001e-07, - "deprecation_date": "2024-08-30", - "input_cost_per_token": 5.0001e-07, - "input_dbu_cost_per_token": 7.143e-06, - "litellm_provider": "databricks", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 0.0, - "output_dbu_cost_per_token": 0.0, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, "databricks/databricks-qwen35-122b-a10b": { "cache_creation_input_token_cost": 2.2001e-07, "cache_read_input_token_cost": 2.2001e-07, @@ -21498,18 +20366,6 @@ "supports_tool_choice": true, "supports_function_calling": true }, - "deepinfra/google/gemini-2.0-flash-001": { - "deprecation_date": "2026-06-01", - "max_tokens": 1000000, - "max_input_tokens": 1000000, - "max_output_tokens": 1000000, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 4e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true, - "supports_function_calling": true - }, "deepinfra/google/gemini-2.5-flash": { "max_tokens": 1000000, "max_input_tokens": 1000000, @@ -22368,15 +21224,6 @@ "/v1/audio/speech" ] }, - "embed-english-light-v2.0": { - "deprecation_date": "2026-04-04", - "input_cost_per_token": 1e-07, - "litellm_provider": "cohere", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, "embed-english-light-v3.0": { "input_cost_per_token": 1e-07, "litellm_provider": "cohere", @@ -22385,15 +21232,6 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, - "embed-english-v2.0": { - "deprecation_date": "2026-04-04", - "input_cost_per_token": 1e-07, - "litellm_provider": "cohere", - "max_input_tokens": 4096, - "max_tokens": 4096, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, "embed-english-v3.0": { "input_cost_per_image": 0.0001, "input_cost_per_token": 1e-07, @@ -22408,15 +21246,6 @@ "supports_embedding_image_input": true, "supports_image_input": true }, - "embed-multilingual-v2.0": { - "deprecation_date": "2026-04-04", - "input_cost_per_token": 1e-07, - "litellm_provider": "cohere", - "max_input_tokens": 768, - "max_tokens": 768, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, "embed-multilingual-v3.0": { "input_cost_per_token": 1e-07, "litellm_provider": "cohere", @@ -22584,23 +21413,6 @@ "cache_read_input_token_cost": 3e-07, "cache_creation_input_token_cost": 3.75e-06 }, - "eu.anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 - }, "eu.anthropic.claude-3-opus-20240229-v1:0": { "input_cost_per_token": 1.5e-05, "litellm_provider": "bedrock", @@ -22616,23 +21428,6 @@ "cache_read_input_token_cost": 1.5e-06, "cache_creation_input_token_cost": 1.875e-05 }, - "eu.anthropic.claude-3-sonnet-20240229-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, "eu.anthropic.claude-opus-4-1-20250805-v1:0": { "cache_creation_input_token_cost": 1.875e-05, "cache_read_input_token_cost": 1.5e-06, @@ -24966,6 +23761,52 @@ "/v1/images/generations" ] }, + "fal_ai/fal-ai/trellis": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://fal.ai/models/fal-ai/trellis", + "metadata": { + "comment": "image-to-3D, returns a GLB mesh; served through the /fal_ai pass-through route" + } + }, + "fal_ai/fal-ai/trellis-2": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.3, + "output_cost_per_image_512": 0.25, + "output_cost_per_image_1024": 0.3, + "output_cost_per_image_1536": 0.35, + "source": "https://fal.ai/models/fal-ai/trellis-2", + "metadata": { + "comment": "image-to-3D, returns a GLB mesh; priced by the request's resolution field (default 1024); served through the /fal_ai pass-through route" + } + }, + "fal_ai/fal-ai/flux-lora-depth": { + "litellm_provider": "fal_ai", + "metadata": { + "notes": "fal bills fal-ai/flux-lora-depth at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries" + }, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-lora-depth", + "supported_endpoints": [ + "/v1/images/edits" + ] + }, + "fal_ai/fal-ai/moondream3-preview/query": { + "input_cost_per_token": 4e-07, + "litellm_provider": "fal_ai", + "mode": "chat", + "output_cost_per_token": 3.5e-06, + "source": "https://fal.ai/models/fal-ai/moondream3-preview/query", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_reasoning": true, + "supports_vision": true + }, "featherless_ai/featherless-ai/Qwerky-72B": { "litellm_provider": "featherless_ai", "max_input_tokens": 32768, @@ -25158,26 +23999,6 @@ "supports_tool_choice": true, "supports_vision": false }, - "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro": { - "cache_read_input_token_cost": 6e-07, - "cache_read_input_token_cost_priority": 6e-07, - "deprecation_date": "2026-08-27", - "input_cost_per_token": 1.2e-06, - "input_cost_per_token_priority": 1.2e-06, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_priority": 1.2e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false - }, "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813": { "cache_read_input_token_cost": 4.4e-08, "cache_read_input_token_cost_priority": 5.5e-08, @@ -25565,26 +24386,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "fireworks_ai/accounts/fireworks/models/minimax-m2p7": { - "cache_read_input_token_cost": 6e-08, - "cache_read_input_token_cost_priority": 6e-07, - "deprecation_date": "2026-08-27", - "input_cost_per_token": 3e-07, - "input_cost_per_token_priority": 1.2e-06, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 196608, - "max_output_tokens": 196608, - "max_tokens": 196608, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_priority": 1.2e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false - }, "fireworks_ai/accounts/fireworks/models/minimax-m3": { "cache_read_input_token_cost": 6e-08, "cache_read_input_token_cost_priority": 9e-08, @@ -25672,26 +24473,6 @@ "supports_tool_choice": true, "supports_vision": false }, - "fireworks_ai/deepseek-v4-pro": { - "cache_read_input_token_cost": 6e-07, - "cache_read_input_token_cost_priority": 6e-07, - "deprecation_date": "2026-08-27", - "input_cost_per_token": 1.2e-06, - "input_cost_per_token_priority": 1.2e-06, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_priority": 1.2e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false - }, "fireworks_ai/glm-4p7": { "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 6e-07, @@ -25891,26 +24672,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "fireworks_ai/minimax-m2p7": { - "cache_read_input_token_cost": 6e-08, - "cache_read_input_token_cost_priority": 6e-07, - "deprecation_date": "2026-08-27", - "input_cost_per_token": 3e-07, - "input_cost_per_token_priority": 1.2e-06, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 196608, - "max_output_tokens": 196608, - "max_tokens": 196608, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_priority": 1.2e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false - }, "fireworks_ai/minimax-m3": { "cache_read_input_token_cost": 6e-08, "cache_read_input_token_cost_priority": 9e-08, @@ -26088,31 +24849,6 @@ "comment": "Open flagship GLM for long-horizon coding agents and million-token context work", "source": "https://api.friendli.ai/serverless/v1/models" }, - "friendliai/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B": { - "litellm_provider": "friendliai", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "input_cost_per_token": 6e-07, - "output_cost_per_token": 2.4e-06, - "cache_read_input_token_cost": 1.2e-07, - "supports_prompt_caching": true, - "supports_reasoning": true, - "reasoning_effort_levels": [], - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_image_input": false, - "supports_video_input": false, - "mode": "chat", - "comment": "Frontier-scale multilingual language model developed by LG AI Research", - "deprecation_date": "2026-09-06", - "source": "https://api.friendli.ai/serverless/v1/models" - }, "friendliai/deepseek-ai/DeepSeek-V3.2": { "litellm_provider": "friendliai", "max_input_tokens": 163840, @@ -26278,6 +25014,7 @@ }, "ft:gpt-4o-2024-08-06": { "cache_read_input_token_cost": 1.875e-06, + "cache_read_input_token_cost_batches": 9e-07, "input_cost_per_token": 3.75e-06, "input_cost_per_token_batches": 2.225e-06, "litellm_provider": "openai", @@ -26316,6 +25053,7 @@ }, "ft:gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "litellm_provider": "openai", @@ -26336,6 +25074,7 @@ }, "ft:gpt-4.1-2025-04-14": { "cache_read_input_token_cost": 7.5e-07, + "cache_read_input_token_cost_batches": 5e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "openai", @@ -26355,6 +25094,7 @@ }, "ft:gpt-4.1-mini-2025-04-14": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_token": 8e-07, "input_cost_per_token_batches": 4e-07, "litellm_provider": "openai", @@ -26374,6 +25114,7 @@ }, "ft:gpt-4.1-nano-2025-04-14": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "deprecation_date": "2026-10-23", "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, @@ -26394,6 +25135,7 @@ }, "ft:o4-mini-2025-04-16": { "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_batches": 5e-07, "deprecation_date": "2026-10-23", "input_cost_per_token": 4e-06, "input_cost_per_token_batches": 2e-06, @@ -26412,160 +25154,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "gemini-2.0-flash": { - "cache_read_input_token_cost": 2.5e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 1e-06, - "input_cost_per_audio_token_batches": 5e-07, - "input_cost_per_character": 3.75e-08, - "input_cost_per_token": 1.5e-07, - "input_cost_per_token_batches": 7.5e-08, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-07, - "output_cost_per_token_batches": 3e-07, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini-2.0-flash-001": { - "cache_read_input_token_cost": 3.75e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 1e-06, - "input_cost_per_token": 1.5e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini-2.0-flash-lite": { - "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7.5e-08, - "input_cost_per_audio_token_batches": 3.75e-08, - "input_cost_per_character": 1.875e-08, - "input_cost_per_token": 7.5e-08, - "input_cost_per_token_batches": 3.75e-08, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "output_cost_per_token_batches": 1.5e-07, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini-2.0-flash-lite-001": { - "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7.5e-08, - "input_cost_per_token": 7.5e-08, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, "gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, "deprecation_date": "2026-10-20", @@ -26677,6 +25265,7 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -26725,7 +25314,8 @@ "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, - "web_search_billing_unit": "per_query" + "web_search_billing_unit": "per_query", + "cache_read_input_token_cost_batches": 1e-07 }, "gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -26769,6 +25359,7 @@ }, "gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -26810,7 +25401,8 @@ "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, - "web_search_billing_unit": "per_query" + "web_search_billing_unit": "per_query", + "cache_read_input_token_cost_batches": 2.5e-08 }, "gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, @@ -26852,6 +25444,7 @@ }, "gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -26889,7 +25482,8 @@ "supports_response_schema": false, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.25e-08 }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, @@ -26944,6 +25538,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -26998,11 +25593,13 @@ }, "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_audio_token_batches": 2.5e-07 + "input_cost_per_audio_token_batches": 2.5e-07, + "cache_read_input_token_cost_batches": 1.25e-08 }, "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -27055,10 +25652,12 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 1.5e-08 }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -27090,7 +25689,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "cache_read_input_token_cost_batches": 1e-07 }, "gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, @@ -27378,53 +25978,6 @@ }, "gemini_native_audio": true }, - "gemini-2.5-flash-lite-preview-06-17": { - "deprecation_date": "2025-11-18", - "cache_read_input_token_cost": 1e-08, - "input_cost_per_audio_token": 5e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_reasoning_token": 4e-07, - "output_cost_per_token": 4e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "google_maps_grounding_cost_per_query": 0.025, - "supports_image_size": false - }, "gemini-2.5-pro": { "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 1.25e-07, @@ -27483,62 +26036,6 @@ "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 1.8e-05 }, - "gemini-3-pro-preview": { - "deprecation_date": "2026-03-26", - "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, - "input_cost_per_token": 2e-06, - "input_cost_per_token_above_200k_tokens": 4e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "output_cost_per_token_above_200k_tokens": 1.8e-05, - "output_cost_per_token_batches": 6e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_video_input": true, - "supports_vision": true, - "supports_web_search": true, - "supports_native_streaming": true, - "input_cost_per_token_priority": 3.6e-06, - "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, - "output_cost_per_token_priority": 2.16e-05, - "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, - "cache_read_input_token_cost_priority": 3.6e-07, - "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query" - }, "gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, @@ -27769,6 +26266,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_token": 1.5e-06, "input_cost_per_audio_token": 1.5e-06, "litellm_provider": "vertex_ai", @@ -27823,11 +26321,13 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_read_input_token_cost_batches": 7.5e-08 }, "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -27880,11 +26380,13 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "vertex_ai/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -27938,11 +26440,13 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "vertex_ai/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -27996,7 +26500,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "vertex_ai/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, @@ -28189,53 +26694,9 @@ "supports_url_context": true, "supports_vision": true }, - "gemini/gemini-robotics-er-1.5-preview": { - "cache_read_input_token_cost": 0, - "deprecation_date": "2026-04-30", - "input_cost_per_token": 3e-07, - "input_cost_per_audio_token": 1e-06, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "output_cost_per_reasoning_token": 2.5e-06, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-robotics-er-1-5-preview", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions" - ], - "supported_modalities": [ - "text", - "image", - "video", - "audio" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": false, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000, - "rpm": 10, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, "gemini/gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_audio_token": 2e-06, "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, @@ -28283,53 +26744,6 @@ "supports_web_search": true, "web_search_billing_unit": "per_query" }, - "gemini/gemini-robotics-er-1.6-preview": { - "deprecation_date": "2026-08-31", - "input_cost_per_audio_token": 2e-06, - "input_cost_per_token": 1e-06, - "litellm_provider": "gemini", - "max_input_tokens": 131072, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_reasoning_token": 5e-06, - "output_cost_per_token": 5e-06, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-robotics-er", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_video_input": true, - "supports_vision": true, - "supports_web_search": true, - "web_search_billing_unit": "per_query" - }, "gemini-2.5-computer-use-preview-10-2025": { "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, @@ -28466,27 +26880,6 @@ "source": "https://ai.google.dev/gemini-api/docs/embeddings#model-versions", "tpm": 10000000 }, - "gemini/gemini-embedding-2-preview": { - "deprecation_date": "2026-08-10", - "input_cost_per_audio_token": 6.5e-06, - "input_cost_per_audio_token_batches": 3.25e-06, - "input_cost_per_image_token": 4.5e-07, - "input_cost_per_image_token_batches": 2.25e-07, - "input_cost_per_token": 2e-07, - "input_cost_per_token_batches": 1e-07, - "input_cost_per_video_token": 1.2e-05, - "input_cost_per_video_token_batches": 6e-06, - "litellm_provider": "gemini", - "max_input_tokens": 8192, - "max_tokens": 8192, - "mode": "embedding", - "output_cost_per_token": 0, - "output_vector_size": 3072, - "rpm": 10000, - "source": "https://ai.google.dev/gemini-api/docs/pricing", - "supports_multimodal": true, - "tpm": 10000000 - }, "gemini/gemini-embedding-2": { "input_cost_per_audio_token": 6.5e-06, "input_cost_per_audio_token_batches": 3.25e-06, @@ -28509,138 +26902,10 @@ "supports_vision": true, "tpm": 10000000 }, - "gemini/gemini-1.5-flash": { - "deprecation_date": "2025-09-29", - "input_cost_per_token": 7.5e-08, - "input_cost_per_token_above_128k_tokens": 1.5e-07, - "litellm_provider": "gemini", - "max_input_tokens": 8192, - "max_tokens": 8192, - "mode": "embedding", - "output_cost_per_token": 0, - "output_vector_size": 3072, - "rpm": 10000, - "source": "https://ai.google.dev/gemini-api/docs/embeddings#multimodal", - "supports_multimodal": true, - "tpm": 10000000 - }, - "gemini/gemini-2.0-flash": { - "cache_read_input_token_cost": 2.5e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4e-07, - "rpm": 10000, - "source": "https://ai.google.dev/pricing#2_0flash", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 10000000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini/gemini-2.0-flash-001": { - "cache_read_input_token_cost": 2.5e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4e-07, - "rpm": 10000, - "source": "https://ai.google.dev/pricing#2_0flash", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 10000000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini/gemini-2.0-flash-lite": { - "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7.5e-08, - "input_cost_per_token": 7.5e-08, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "rpm": 4000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.0-flash-lite", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 4000000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, "gemini/gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_audio_token": 1e-06, @@ -28799,50 +27064,6 @@ "web_search_billing_unit": "per_query", "supports_reasoning": false }, - "gemini/gemini-3-pro-image-preview": { - "deprecation_date": "2026-06-25", - "input_cost_per_image": 0.0011, - "input_cost_per_token": 2e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "gemini", - "max_input_tokens": 65536, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "image_generation", - "output_cost_per_image": 0.134, - "output_cost_per_image_token": 0.00012, - "output_cost_per_token": 1.2e-05, - "rpm": 1000, - "tpm": 4000000, - "output_cost_per_token_batches": 6e-06, - "source": "https://ai.google.dev/gemini-api/docs/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_prompt_caching": true, - "supports_reasoning": false, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query" - }, "gemini/nano-banana-pro-preview": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -28928,49 +27149,6 @@ }, "web_search_billing_unit": "per_query" }, - "gemini/gemini-3.1-flash-image-preview": { - "deprecation_date": "2026-06-25", - "input_cost_per_token": 5e-07, - "input_cost_per_token_batches": 2.5e-07, - "litellm_provider": "gemini", - "max_input_tokens": 65536, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "image_generation", - "output_cost_per_image": 0.045, - "output_cost_per_image_token": 6e-05, - "output_cost_per_token": 3e-06, - "output_cost_per_token_batches": 1.5e-06, - "rpm": 1000, - "tpm": 4000000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-image-preview", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_prompt_caching": true, - "supports_reasoning": false, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query" - }, "gemini/gemini-3.1-flash-lite-image": { "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -29052,6 +27230,7 @@ "gemini/gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, "input_cost_per_audio_token": 3e-07, @@ -29108,104 +27287,6 @@ "supports_audio_input": true, "supports_image_size": false }, - "gemini/gemini-2.5-flash-lite-preview-09-2025": { - "cache_read_input_token_cost": 1e-08, - "deprecation_date": "2026-03-31", - "input_cost_per_audio_token": 3e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_reasoning_token": 4e-07, - "output_cost_per_token": 4e-07, - "rpm": 15, - "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "google_maps_grounding_cost_per_query": 0.025, - "supports_image_size": false - }, - "gemini/gemini-2.5-flash-preview-09-2025": { - "cache_read_input_token_cost": 3e-08, - "deprecation_date": "2026-02-17", - "input_cost_per_audio_token": 1e-06, - "input_cost_per_token": 3e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_reasoning_token": 2.5e-06, - "output_cost_per_token": 2.5e-06, - "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "google_maps_grounding_cost_per_query": 0.025, - "supports_image_size": false - }, "gemini/gemini-flash-latest": { "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, @@ -29323,55 +27404,6 @@ "supports_video_input": true, "web_search_billing_unit": "per_query" }, - "gemini/gemini-2.5-flash-lite-preview-06-17": { - "deprecation_date": "2025-11-18", - "cache_read_input_token_cost": 1e-08, - "input_cost_per_audio_token": 5e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_reasoning_token": 4e-07, - "output_cost_per_token": 4e-07, - "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "google_maps_grounding_cost_per_query": 0.025, - "supports_image_size": false - }, "gemini/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, @@ -29397,6 +27429,7 @@ "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, "cache_read_input_token_cost_flex": 1.25e-07, "cache_read_input_token_cost_priority": 2.25e-07, "input_cost_per_token": 1.25e-06, @@ -29481,117 +27514,10 @@ "supports_vision": true, "tpm": 800000 }, - "gemini/gemini-3-pro-preview": { - "deprecation_date": "2026-03-09", - "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "input_cost_per_token": 2e-06, - "input_cost_per_token_above_200k_tokens": 4e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "output_cost_per_token_above_200k_tokens": 1.8e-05, - "output_cost_per_token_batches": 6e-06, - "rpm": 2000, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_video_input": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 800000, - "input_cost_per_token_priority": 3.6e-06, - "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, - "output_cost_per_token_priority": 2.16e-05, - "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, - "cache_read_input_token_cost_priority": 3.6e-07, - "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query" - }, - "gemini/gemini-3.1-flash-lite-preview": { - "cache_read_input_token_cost": 2.5e-08, - "deprecation_date": "2026-05-25", - "input_cost_per_audio_token": 5e-07, - "input_cost_per_token": 2.5e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_reasoning_token": 1.5e-06, - "output_cost_per_token": 1.5e-06, - "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/models", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_video_input": true, - "supports_vision": true, - "supports_web_search": true, - "supports_native_streaming": true, - "tpm": 250000, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 - }, "gemini/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-05-07", @@ -29653,6 +27579,7 @@ }, "gemini/gemini-3.5-flash-lite": { "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 2e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, "input_cost_per_token": 3e-07, @@ -29712,6 +27639,7 @@ "gemini/gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 5e-07, @@ -29773,6 +27701,7 @@ "gemini/gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "gemini", @@ -29834,6 +27763,7 @@ "gemini/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -29893,6 +27823,7 @@ "gemini/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -29953,6 +27884,7 @@ "gemini/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30095,6 +28027,7 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 2e-07, "cache_read_input_token_cost_flex": 2e-07, "search_context_cost_per_query": { "search_context_size_low": 0.014, @@ -30157,6 +28090,7 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 2e-07, "cache_read_input_token_cost_flex": 2e-07, "search_context_cost_per_query": { "search_context_size_low": 0.014, @@ -30261,6 +28195,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", @@ -30315,11 +28250,13 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_read_input_token_cost_batches": 7.5e-08 }, "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30372,11 +28309,13 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30430,11 +28369,13 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30488,7 +28429,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, @@ -30672,34 +28614,6 @@ "output_cost_per_image": 0.04, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, - "gemini/imagen-3.0-generate-002": { - "deprecation_date": "2025-11-10", - "litellm_provider": "gemini", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "gemini/imagen-4.0-fast-generate-001": { - "deprecation_date": "2026-08-17", - "litellm_provider": "gemini", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "gemini/imagen-4.0-generate-001": { - "deprecation_date": "2026-08-17", - "litellm_provider": "gemini", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "gemini/imagen-4.0-ultra-generate-001": { - "deprecation_date": "2026-08-17", - "litellm_provider": "gemini", - "mode": "image_generation", - "output_cost_per_image": 0.06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, "gemini/learnlm-1.5-pro-experimental": { "input_cost_per_audio_per_second": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -30778,21 +28692,6 @@ "supports_web_search": false, "output_cost_per_image": 0.08 }, - "gemini/veo-2.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "gemini", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "video_generation", - "output_cost_per_second": 0.35, - "source": "https://ai.google.dev/gemini-api/docs/video", - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "video" - ] - }, "gemini/veo-3.1-fast-generate-preview": { "litellm_provider": "gemini", "max_input_tokens": 1024, @@ -31652,10 +29551,21 @@ "output_cost_per_token": 3.15e-06 }, "baseten/zai-org/GLM-4.7": { + "cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 6e-07, "litellm_provider": "baseten", + "max_input_tokens": 200000, + "max_output_tokens": 200000, + "max_tokens": 200000, "mode": "chat", - "output_cost_per_token": 2.2e-06 + "output_cost_per_token": 2.2e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false }, "baseten/zai-org/GLM-4.6": { "input_cost_per_token": 6e-07, @@ -31682,10 +29592,21 @@ "output_cost_per_token": 2.5e-06 }, "baseten/openai/gpt-oss-120b": { + "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-07, "litellm_provider": "baseten", + "max_input_tokens": 128072, + "max_output_tokens": 128072, + "max_tokens": 128072, "mode": "chat", - "output_cost_per_token": 5e-07 + "output_cost_per_token": 5e-07, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false }, "baseten/deepseek-ai/DeepSeek-V3.1": { "input_cost_per_token": 5e-07, @@ -31980,33 +29901,6 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "gpt-4-0125-preview": { - "deprecation_date": "2026-03-26", - "input_cost_per_token": 1e-05, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 3e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, - "gpt-4-0314": { - "deprecation_date": "2026-03-26", - "input_cost_per_token": 3e-05, - "litellm_provider": "openai", - "max_input_tokens": 8192, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-05, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4-0613": { "deprecation_date": "2026-10-23", "input_cost_per_token": 3e-05, @@ -32076,22 +29970,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "gpt-4-turbo-preview": { - "deprecation_date": "2026-03-26", - "input_cost_per_token": 1e-05, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 3e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4.1": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_priority": 8.75e-07, @@ -32434,24 +30312,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "gpt-4o-audio-preview": { - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 2.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_token": 1e-05, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4o-audio-preview-2024-12-17": { "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 4e-05, @@ -32636,44 +30496,6 @@ "supports_tool_choice": true, "supports_vision": false }, - "gpt-audio-mini-2025-10-06": { - "deprecation_date": "2026-07-23", - "input_cost_per_audio_token": 1e-05, - "input_cost_per_token": 6e-07, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 2.4e-06, - "source": "https://developers.openai.com/api/docs/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses", - "/v1/realtime", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "audio" - ], - "supported_output_modalities": [ - "text", - "audio" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": false, - "supports_reasoning": false, - "supports_response_schema": false, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": false - }, "gpt-audio-mini-2025-12-15": { "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, @@ -32770,24 +30592,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "gpt-4o-mini-audio-preview": { - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 1e-05, - "input_cost_per_token": 1.5e-07, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 6e-07, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4o-mini-audio-preview-2024-12-17": { "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 1e-05, @@ -32806,26 +30610,6 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "gpt-4o-mini-realtime-preview": { - "cache_creation_input_audio_token_cost": 3e-07, - "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 1e-05, - "input_cost_per_token": 6e-07, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 2.4e-06, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4o-mini-realtime-preview-2024-12-17": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 3e-07, @@ -32872,32 +30656,6 @@ "supports_vision": true, "supports_web_search": true }, - "gpt-4o-mini-search-preview-2025-03-11": { - "cache_read_input_token_cost": 7.5e-08, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.5e-07, - "input_cost_per_token_batches": 7.5e-08, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 6e-07, - "output_cost_per_token_batches": 3e-07, - "search_context_cost_per_query": { - "search_context_size_high": 0.025, - "search_context_size_low": 0.025, - "search_context_size_medium": 0.025 - }, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "gpt-4o-mini-transcribe": { "input_cost_per_audio_token": 1.25e-06, "input_cost_per_token": 1.25e-06, @@ -32932,63 +30690,6 @@ "audio" ] }, - "gpt-4o-realtime-preview": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_token": 2e-05, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, - "gpt-4o-realtime-preview-2024-12-17": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_token": 2e-05, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, - "gpt-4o-realtime-preview-2025-06-03": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_token": 2e-05, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4o-search-preview": { "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, @@ -33015,32 +30716,6 @@ "supports_vision": true, "supports_web_search": true }, - "gpt-4o-search-preview-2025-03-11": { - "cache_read_input_token_cost": 1.25e-06, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 2.5e-06, - "input_cost_per_token_batches": 1.25e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "output_cost_per_token_batches": 5e-06, - "search_context_cost_per_query": { - "search_context_size_high": 0.025, - "search_context_size_low": 0.025, - "search_context_size_medium": 0.025 - }, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "gpt-4o-transcribe": { "input_cost_per_audio_token": 2.5e-06, "input_cost_per_token": 2.5e-06, @@ -33058,6 +30733,7 @@ }, "gpt-image-1.5": { "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", "input_cost_per_token": 5e-06, "litellm_provider": "openai", @@ -33077,6 +30753,7 @@ }, "gpt-image-1.5-2025-12-16": { "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", "input_cost_per_token": 5e-06, "litellm_provider": "openai", @@ -33096,6 +30773,7 @@ }, "gpt-image-2": { "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_batches": 6.25e-07, "input_cost_per_token": 5e-06, "litellm_provider": "openai", "mode": "image_generation", @@ -33547,6 +31225,7 @@ }, "gpt-5": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -33597,6 +31276,7 @@ }, "gpt-5.1": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -33648,6 +31328,7 @@ }, "gpt-5.1-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -33697,54 +31378,9 @@ "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": false }, - "gpt-5.1-chat-latest": { - "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_priority": 2.5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "input_cost_per_token_priority": 2.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "output_cost_per_token_priority": 2e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_native_streaming": true, - "supports_parallel_function_calling": false, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": false, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, "gpt-5.2": { "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_batches": 8.75e-08, "cache_read_input_token_cost_flex": 8.75e-08, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, @@ -33797,6 +31433,7 @@ }, "gpt-5.2-2025-12-11": { "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_batches": 8.75e-08, "cache_read_input_token_cost_flex": 8.75e-08, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, @@ -33847,94 +31484,6 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, - "gpt-5.2-chat-latest": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-08-10", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.3-chat-latest": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-08-10", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, "gpt-5.2-pro": { "input_cost_per_token": 2.1e-05, "input_cost_per_token_batches": 1.05e-05, @@ -34030,6 +31579,10 @@ "cache_read_input_token_cost_above_272k_tokens": 2e-06, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06, + "cache_read_input_token_cost_batches": 5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 1e-06, + "cache_creation_input_token_cost_batches": 6.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-05, "cache_read_input_token_cost_flex": 5e-07, "cache_read_input_token_cost_priority": 2e-06, "input_cost_per_token": 1e-05, @@ -34037,6 +31590,7 @@ "input_cost_per_token_above_272k_tokens_flex": 1e-05, "input_cost_per_token_above_272k_tokens_priority": 4e-05, "input_cost_per_token_batches": 5e-06, + "input_cost_per_token_above_272k_tokens_batches": 1e-05, "input_cost_per_token_flex": 5e-06, "input_cost_per_token_priority": 2e-05, "litellm_provider": "openai", @@ -34049,6 +31603,7 @@ "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, "output_cost_per_token_above_272k_tokens_priority": 0.00015, "output_cost_per_token_batches": 2.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 3.75e-05, "output_cost_per_token_flex": 2.5e-05, "output_cost_per_token_priority": 0.0001, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34089,6 +31644,158 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-6-sol": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, + "cache_creation_input_token_cost_flex": 1.25e-06, + "cache_creation_input_token_cost_priority": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_read_input_token_cost_batches": 1e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2e-07, + "cache_creation_input_token_cost_batches": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, + "cache_read_input_token_cost_flex": 1e-07, + "cache_read_input_token_cost_priority": 4e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "input_cost_per_token_above_272k_tokens_priority": 8e-06, + "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_above_272k_tokens_batches": 2e-06, + "input_cost_per_token_flex": 1e-06, + "input_cost_per_token_priority": 4e-06, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_272k_tokens": 1.5e-05, + "output_cost_per_token_above_272k_tokens_flex": 7.5e-06, + "output_cost_per_token_above_272k_tokens_priority": 3e-05, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_above_272k_tokens_batches": 7.5e-06, + "output_cost_per_token_flex": 5e-06, + "output_cost_per_token_priority": 2e-05, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, + "gpt-6-luna": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07, + "cache_creation_input_token_cost_flex": 6.25e-08, + "cache_creation_input_token_cost_priority": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 4e-08, + "cache_read_input_token_cost_batches": 5e-09, + "cache_read_input_token_cost_above_272k_tokens_batches": 1e-08, + "cache_creation_input_token_cost_batches": 6.25e-08, + "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-07, + "cache_read_input_token_cost_flex": 5e-09, + "cache_read_input_token_cost_priority": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "input_cost_per_token_above_272k_tokens_flex": 1e-07, + "input_cost_per_token_above_272k_tokens_priority": 4e-07, + "input_cost_per_token_batches": 5e-08, + "input_cost_per_token_above_272k_tokens_batches": 1e-07, + "input_cost_per_token_flex": 5e-08, + "input_cost_per_token_priority": 2e-07, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_above_272k_tokens": 7.5e-07, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-07, + "output_cost_per_token_above_272k_tokens_priority": 1.5e-06, + "output_cost_per_token_batches": 2.5e-07, + "output_cost_per_token_above_272k_tokens_batches": 3.75e-07, + "output_cost_per_token_flex": 2.5e-07, + "output_cost_per_token_priority": 1e-06, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, @@ -34100,6 +31807,10 @@ "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_read_input_token_cost_batches": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 4e-07, + "cache_creation_input_token_cost_batches": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 5e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, @@ -34107,6 +31818,7 @@ "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 4e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", @@ -34119,6 +31831,7 @@ "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.5e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34167,6 +31880,10 @@ "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_read_input_token_cost_batches": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 4e-07, + "cache_creation_input_token_cost_batches": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 5e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, @@ -34174,6 +31891,7 @@ "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 4e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", @@ -34186,6 +31904,7 @@ "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.5e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34236,6 +31955,10 @@ "cache_read_input_token_cost_above_272k_tokens": 4e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_read_input_token_cost_batches": 1e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2e-07, + "cache_creation_input_token_cost_batches": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 4e-07, "input_cost_per_token": 2e-06, @@ -34243,6 +31966,7 @@ "input_cost_per_token_above_272k_tokens_flex": 2e-06, "input_cost_per_token_above_272k_tokens_priority": 8e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_above_272k_tokens_batches": 2e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, "litellm_provider": "openai", @@ -34255,6 +31979,7 @@ "output_cost_per_token_above_272k_tokens_flex": 9e-06, "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, "output_cost_per_token_batches": 6e-06, + "output_cost_per_token_above_272k_tokens_batches": 9e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34304,6 +32029,10 @@ "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-08, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, + "cache_read_input_token_cost_batches": 1e-08, + "cache_read_input_token_cost_above_272k_tokens_batches": 2e-08, + "cache_creation_input_token_cost_batches": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -34311,6 +32040,7 @@ "input_cost_per_token_above_272k_tokens_flex": 2e-07, "input_cost_per_token_above_272k_tokens_priority": 8e-07, "input_cost_per_token_batches": 1e-07, + "input_cost_per_token_above_272k_tokens_batches": 2e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_priority": 4e-07, "litellm_provider": "openai", @@ -34323,6 +32053,7 @@ "output_cost_per_token_above_272k_tokens_flex": 9e-07, "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_above_272k_tokens_batches": 9e-07, "output_cost_per_token_flex": 6e-07, "output_cost_per_token_priority": 2.4e-06, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34593,12 +32324,15 @@ "gpt-5.5": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost_batches": 2.5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34609,6 +32343,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34651,12 +32386,15 @@ "gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost_batches": 2.5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34667,6 +32405,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34711,6 +32450,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -34720,6 +32460,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -34760,6 +32501,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -34769,6 +32511,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -34807,12 +32550,15 @@ "gpt-5.4": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_batches": 1.3e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_priority": 5e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34823,6 +32569,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_priority": 3e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34860,12 +32607,15 @@ "gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_batches": 1.3e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_priority": 5e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34876,6 +32626,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_priority": 3e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34915,6 +32666,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -34924,6 +32676,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -34965,6 +32718,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -34974,6 +32728,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -35012,6 +32767,7 @@ }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -35065,6 +32821,7 @@ }, "gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -35118,6 +32875,7 @@ }, "gpt-5.4-nano": { "cache_read_input_token_cost": 2e-08, + "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "input_cost_per_token": 2e-07, "input_cost_per_token_flex": 1e-07, @@ -35168,6 +32926,7 @@ }, "gpt-5.4-nano-2026-03-17": { "cache_read_input_token_cost": 2e-08, + "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "input_cost_per_token": 2e-07, "input_cost_per_token_flex": 1e-07, @@ -35303,6 +33062,7 @@ }, "gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", @@ -35350,7 +33110,8 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_batches": 6.25e-08 }, "gpt-5-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -35387,251 +33148,6 @@ "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": true }, - "gpt-5-chat-latest": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": false, - "supports_native_streaming": true, - "supports_parallel_function_calling": false, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": false, - "supports_vision": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5-codex": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 1e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.1-codex": { - "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_priority": 2.5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "input_cost_per_token_priority": 2.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 1e-05, - "output_cost_per_token_priority": 2e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.1-codex-max": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 1e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.1-codex-mini": { - "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_priority": 4.5e-08, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 2.5e-07, - "input_cost_per_token_priority": 4.5e-07, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 2e-06, - "output_cost_per_token_priority": 3.6e-06, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.2-codex": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true - }, "gpt-5.3-codex": { "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, @@ -35677,6 +33193,7 @@ }, "gpt-5-mini": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_token": 2.5e-07, @@ -35727,6 +33244,7 @@ }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", @@ -35774,10 +33292,12 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_batches": 1.25e-08 }, "gpt-5-nano": { "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, @@ -35826,6 +33346,7 @@ }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-08, @@ -35871,10 +33392,12 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_batches": 2.5e-09 }, "gpt-image-1": { "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-10-23", "input_cost_per_image_token": 1e-05, "input_cost_per_image_token_batches": 5e-06, @@ -35891,6 +33414,7 @@ }, "gpt-image-1-mini": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "deprecation_date": "2026-12-01", "input_cost_per_image_token": 2.5e-06, "input_cost_per_image_token_batches": 1.25e-06, @@ -36466,44 +33990,14 @@ "supports_response_schema": true, "supports_vision": true }, - "groq/llama-3.1-8b-instant": { - "deprecation_date": "2026-08-16", - "input_cost_per_token": 5e-08, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 8e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.3-70b-versatile": { - "deprecation_date": "2026-08-16", - "input_cost_per_token": 5.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 7.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/gemma-7b-it": { - "deprecation_date": "2024-12-18", - "input_cost_per_token": 5e-08, + "groq/llama-guard-3-8b": { + "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 8192, - "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 8e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true + "output_cost_per_token": 2e-07, + "source": "https://console.groq.com/docs/model/llama-guard-3-8b" }, "groq/meta-llama/llama-prompt-guard-2-22m": { "input_cost_per_token": 3e-08, @@ -36525,58 +34019,6 @@ "output_cost_per_token": 4e-08, "source": "https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m" }, - "groq/meta-llama/llama-guard-4-12b": { - "deprecation_date": "2026-03-05", - "input_cost_per_token": 2e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-07 - }, - "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { - "deprecation_date": "2026-03-09", - "input_cost_per_token": 2e-07, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/meta-llama/llama-4-scout-17b-16e-instruct": { - "deprecation_date": "2026-07-17", - "input_cost_per_token": 1.1e-07, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3.4e-07, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/moonshotai/kimi-k2-instruct-0905": { - "deprecation_date": "2026-04-15", - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3e-06, - "cache_read_input_token_cost": 5e-07, - "litellm_provider": "groq", - "max_input_tokens": 262144, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "groq/openai/gpt-oss-120b": { "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.5e-07, @@ -36657,45 +34099,6 @@ "mode": "audio_speech", "source": "https://console.groq.com/docs/models" }, - "groq/playai-tts": { - "deprecation_date": "2025-12-31", - "input_cost_per_character": 5e-05, - "litellm_provider": "groq", - "max_input_tokens": 10000, - "max_output_tokens": 10000, - "max_tokens": 10000, - "mode": "audio_speech" - }, - "groq/qwen/qwen3.6-27b": { - "input_cost_per_token": 6e-07, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://console.groq.com/docs/model/qwen/qwen3.6-27b", - "deprecation_date": "2026-09-14", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/qwen/qwen3-32b": { - "deprecation_date": "2026-07-17", - "input_cost_per_token": 2.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 131000, - "max_output_tokens": 131000, - "max_tokens": 131000, - "mode": "chat", - "output_cost_per_token": 5.9e-07, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, "groq/whisper-large-v3": { "input_cost_per_second": 3.083e-05, "litellm_provider": "groq", @@ -36708,27 +34111,6 @@ "mode": "audio_transcription", "output_cost_per_second": 0.0 }, - "hd/1024-x-1024/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 7.629e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, - "hd/1024-x-1792/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 6.539e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, - "hd/1792-x-1024/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 6.539e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "heroku/claude-3-5-haiku": { "litellm_provider": "heroku", "max_tokens": 8192, @@ -38555,19 +35937,6 @@ "supports_system_messages": true, "supports_native_structured_output": true }, - "mistral/codestral-2405": { - "deprecation_date": "2025-06-16", - "input_cost_per_token": 1e-06, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 3e-06, - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/codestral-2508": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, @@ -38611,51 +35980,6 @@ "supports_assistant_prefill": true, "supports_tool_choice": true }, - "mistral/devstral-medium-2507": { - "deprecation_date": "2026-05-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://mistral.ai/news/devstral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/devstral-small-2505": { - "deprecation_date": "2025-11-30", - "input_cost_per_token": 1e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://mistral.ai/news/devstral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/devstral-small-2507": { - "deprecation_date": "2026-05-31", - "input_cost_per_token": 1e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://mistral.ai/news/devstral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/devstral-small-latest": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, @@ -38671,21 +35995,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/labs-devstral-small-2512": { - "deprecation_date": "2026-03-31", - "input_cost_per_token": 1e-07, - "litellm_provider": "mistral", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://docs.mistral.ai/models/devstral-small-2-25-12", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/devstral-latest": { "cache_read_input_token_cost": 4e-08, "input_cost_per_token": 4e-07, @@ -38716,21 +36025,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/devstral-2512": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://mistral.ai/news/devstral-2-vibe-cli", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/ministral-14b-2512": { "input_cost_per_token": 2e-07, "litellm_provider": "mistral", @@ -39006,54 +36300,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/magistral-medium-2506": { - "deprecation_date": "2025-11-30", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/magistral-medium-2509": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/magistral-medium-1-2-2509": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/mistral-ocr-latest": { "litellm_provider": "mistral", "ocr_cost_per_page": 0.004, @@ -39093,20 +36339,6 @@ "/v1/batch" ] }, - "mistral/mistral-ocr-2505-completion": { - "deprecation_date": "2026-05-31", - "litellm_provider": "mistral", - "ocr_cost_per_page": 0.001, - "ocr_cost_per_page_batches": 0.0005, - "annotation_cost_per_page": 0.003, - "annotation_cost_per_page_batches": 0.0015, - "mode": "ocr", - "supported_endpoints": [ - "/v1/ocr", - "/v1/batch" - ], - "source": "https://mistral.ai/pricing#api-pricing" - }, "mistral/mistral-ocr-2512": { "litellm_provider": "mistral", "ocr_cost_per_page": 0.002, @@ -39137,22 +36369,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/magistral-small-2506": { - "deprecation_date": "2025-11-30", - "input_cost_per_token": 5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://mistral.ai/pricing#api-pricing", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/magistral-small-latest": { "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, @@ -39170,22 +36386,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/magistral-small-1-2-2509": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://mistral.ai/pricing#api-pricing", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/mistral-embed": { "input_cost_per_token": 1e-07, "litellm_provider": "mistral", @@ -39209,48 +36409,6 @@ "max_tokens": 8192, "mode": "embedding" }, - "mistral/mistral-large-2402": { - "deprecation_date": "2025-06-16", - "input_cost_per_token": 4e-06, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/mistral-large-2407": { - "deprecation_date": "2025-03-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 9e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/mistral-large-2411": { - "deprecation_date": "2026-05-31", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 6e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/mistral-large-latest": { "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 5e-07, @@ -39320,49 +36478,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/mistral-medium-2312": { - "deprecation_date": "2025-06-16", - "input_cost_per_token": 2.7e-06, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 8.1e-06, - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/mistral-medium-2505": { - "deprecation_date": "2026-08-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 2e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/mistral-medium-2508": { - "deprecation_date": "2026-08-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://mistral.ai/news/mistral-medium-3", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "mistral/mistral-medium-2604": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, @@ -39405,22 +36520,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/mistral-medium-3-1-2508": { - "deprecation_date": "2026-08-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://mistral.ai/news/mistral-medium-3", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "mistral/mistral-medium-3-5": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, @@ -39477,22 +36576,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/mistral-small-3-2-2506": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 6e-08, - "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "source": "https://mistral.ai/pricing", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "mistral/ministral-3-3b-2512": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, @@ -39586,32 +36669,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/open-codestral-mamba": { - "deprecation_date": "2025-06-06", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2.5e-07, - "source": "https://mistral.ai/technology/", - "supports_assistant_prefill": true, - "supports_tool_choice": true - }, - "mistral/open-mistral-7b": { - "deprecation_date": "2025-03-30", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 2.5e-07, - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/open-mistral-nemo": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, @@ -39626,78 +36683,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/open-mistral-nemo-2407": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 3e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://mistral.ai/technology/", - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/open-mixtral-8x22b": { - "deprecation_date": "2025-03-30", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 65336, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 6e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/open-mixtral-8x7b": { - "deprecation_date": "2025-03-30", - "input_cost_per_token": 7e-07, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 7e-07, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/pixtral-12b-2409": { - "deprecation_date": "2025-12-31", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.5e-07, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "mistral/pixtral-large-2411": { - "deprecation_date": "2026-05-31", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 6e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "mistral/pixtral-large-latest": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 2e-06, @@ -39746,36 +36731,6 @@ "supports_audio_input": false, "supports_response_schema": true }, - "moonshot/kimi-k2-0711-preview": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 6e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "moonshot/kimi-k2-0905-preview": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 6e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, "moonshot/kimi-k2.7-code": { "cache_read_input_token_cost": 1.9e-07, "input_cost_per_token": 9.5e-07, @@ -39794,21 +36749,6 @@ "supports_video_input": true, "supports_vision": true }, - "moonshot/kimi-k2-turbo-preview": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 1.15e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 8e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, "moonshot/kimi-k2.5": { "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 6e-07, @@ -39865,111 +36805,6 @@ "supports_video_input": true, "supports_vision": true }, - "moonshot/kimi-latest": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-01-28", - "input_cost_per_token": 2e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "moonshot/kimi-latest-128k": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-01-28", - "input_cost_per_token": 2e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "moonshot/kimi-latest-32k": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-01-28", - "input_cost_per_token": 1e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "moonshot/kimi-latest-8k": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-01-28", - "input_cost_per_token": 2e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "moonshot/kimi-thinking-preview": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2025-11-11", - "input_cost_per_token": 6e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_vision": true - }, - "moonshot/kimi-k2-thinking": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 6e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "moonshot/kimi-k2-thinking-turbo": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 1.15e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 8e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_web_search": true - }, "moonshot/moonshot-v1-128k": { "input_cost_per_token": 2e-06, "litellm_provider": "moonshot", @@ -39983,19 +36818,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "moonshot/moonshot-v1-128k-0430": { - "deprecation_date": "2024-04-30", - "input_cost_per_token": 2e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true - }, "moonshot/moonshot-v1-128k-vision-preview": { "input_cost_per_token": 2e-06, "litellm_provider": "moonshot", @@ -40023,19 +36845,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "moonshot/moonshot-v1-32k-0430": { - "deprecation_date": "2024-04-30", - "input_cost_per_token": 1e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true - }, "moonshot/moonshot-v1-32k-vision-preview": { "input_cost_per_token": 1e-06, "litellm_provider": "moonshot", @@ -40063,19 +36872,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "moonshot/moonshot-v1-8k-0430": { - "deprecation_date": "2024-04-30", - "input_cost_per_token": 2e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true - }, "moonshot/moonshot-v1-8k-vision-preview": { "input_cost_per_token": 2e-07, "litellm_provider": "moonshot", @@ -41266,88 +38062,6 @@ "supports_vision": true, "supports_web_search": true }, - "o3-deep-research": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1e-05, - "input_cost_per_token_batches": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 4e-05, - "output_cost_per_token_batches": 2e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, - "o3-deep-research-2025-06-26": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1e-05, - "input_cost_per_token_batches": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 4e-05, - "output_cost_per_token_batches": 2e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, "o3-mini": { "cache_read_input_token_cost": 5.5e-07, "deprecation_date": "2026-10-23", @@ -41533,88 +38247,6 @@ "supports_vision": true, "supports_web_search": true }, - "o4-mini-deep-research": { - "cache_read_input_token_cost": 5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 2e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 8e-06, - "output_cost_per_token_batches": 4e-06, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, - "o4-mini-deep-research-2025-06-26": { - "cache_read_input_token_cost": 5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 2e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 8e-06, - "output_cost_per_token_batches": 4e-06, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, "oci/meta.llama-3.1-8b-instruct": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", @@ -42587,32 +39219,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/anthropic/claude-opus-4": { - "input_cost_per_image": 0.0048, - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_pdf_input": true, - "supports_response_schema": false, - "supports_web_search": true - }, "openrouter/anthropic/claude-opus-4.1": { "input_cost_per_image": 0.0048, "cache_creation_input_token_cost": 1.875e-05, @@ -42866,6 +39472,28 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "openrouter/anthropic/claude-opus-5.5": { + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "openrouter/bytedance/ui-tars-1.5-7b": { "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-07, @@ -43037,21 +39665,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.5526e-07, + "input_cost_per_token": 8.92272e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.91052e-06, + "output_cost_per_token": 1.784544e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.9605e-08, + "cache_read_input_token_cost": 7.4356e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -43079,44 +39707,27 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro-0813": { - "input_cost_per_token": 5.58624e-07, + "input_cost_per_token": 1.32e-06, "input_cost_per_token_cache_hit": 1.9272e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.675872e-06, + "output_cost_per_token": 3.96e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 1.86208e-08, - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.58624e-7,"output_cost_per_token":0.000001675872,"cache_read_input_token_cost":1.86208e-8}, + "cache_read_input_token_cost": 4.4e-08, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":6.6e-7,"output_cost_per_token":0.00000198,"cache_read_input_token_cost":2.2e-8}, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, "supports_web_search": false }, - "openrouter/google/gemini-2.0-flash-001": { - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4e-07, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/google/gemini-2.5-flash": { "cache_creation_input_token_cost": 8.33333333333333e-08, "cache_read_input_audio_token_cost": 1e-07, @@ -43540,27 +40151,6 @@ "supports_vision": true, "supports_web_search": false }, - "openrouter/mistralai/mistral-large-2512": { - "cache_read_input_token_cost": 5.5e-08, - "input_cost_per_image": 0, - "input_cost_per_token": 5.5e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 209715, - "max_tokens": 209715, - "mode": "chat", - "output_cost_per_token": 1.65e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": false, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, "openrouter/mistralai/mistral-7b-instruct": { "input_cost_per_token": 1.3e-07, "litellm_provider": "openrouter", @@ -44178,19 +40768,19 @@ }, "openrouter/openai/gpt-oss-20b": { "cache_read_input_token_cost": 3e-08, - "input_cost_per_token": 3e-08, + "input_cost_per_token": 1.8e-08, "litellm_provider": "openrouter", "max_input_tokens": 131072, - "max_output_tokens": 117964, - "max_tokens": 117964, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 1.3e-07, + "output_cost_per_token": 9e-08, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": false, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -44812,26 +41402,6 @@ "max_tokens": 128000, "mode": "chat" }, - "openrouter/stealth/union-alpha": { - "deprecation_date": "2098-12-31", - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": false, - "supports_reasoning": false, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_vision": true, - "supports_web_search": false - }, "ovhcloud/DeepSeek-R1-Distill-Llama-70B": { "input_cost_per_token": 6.7e-07, "litellm_provider": "ovhcloud", @@ -46331,17 +42901,6 @@ "supports_reasoning": true, "supports_system_messages": true }, - "rerank-english-v2.0": { - "input_cost_per_query": 0.002, - "input_cost_per_token": 0.0, - "litellm_provider": "cohere", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "rerank", - "output_cost_per_token": 0.0, - "deprecation_date": "2025-04-30" - }, "rerank-english-v3.0": { "input_cost_per_query": 0.002, "input_cost_per_token": 0.0, @@ -46352,17 +42911,6 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, - "rerank-multilingual-v2.0": { - "input_cost_per_query": 0.002, - "input_cost_per_token": 0.0, - "litellm_provider": "cohere", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "rerank", - "output_cost_per_token": 0.0, - "deprecation_date": "2025-04-30" - }, "rerank-multilingual-v3.0": { "input_cost_per_query": 0.002, "input_cost_per_token": 0.0, @@ -46503,31 +43051,6 @@ "output_cost_per_token": 7e-06, "source": "https://cloud.sambanova.ai/plans/pricing" }, - "sambanova/DeepSeek-R1-Distill-Llama-70B": { - "deprecation_date": "2026-03-20", - "input_cost_per_token": 7e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.4e-06, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, - "sambanova/DeepSeek-V3-0324": { - "deprecation_date": "2026-04-14", - "input_cost_per_token": 3e-06, - "litellm_provider": "sambanova", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 4.5e-06, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "sambanova/Llama-4-Maverick-17B-128E-Instruct": { "input_cost_per_token": 6.3e-07, "litellm_provider": "sambanova", @@ -46545,73 +43068,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "sambanova/Llama-4-Scout-17B-16E-Instruct": { - "deprecation_date": "2025-06-19", - "input_cost_per_token": 4e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "metadata": { - "notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount" - }, - "mode": "chat", - "output_cost_per_token": 7e-07, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "sambanova/Meta-Llama-3.1-405B-Instruct": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 5e-06, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "sambanova/Meta-Llama-3.1-8B-Instruct": { - "deprecation_date": "2026-04-14", - "input_cost_per_token": 1e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "sambanova/Meta-Llama-3.2-1B-Instruct": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 4e-08, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 8e-08, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, - "sambanova/Meta-Llama-3.2-3B-Instruct": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 8e-08, - "litellm_provider": "sambanova", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.6e-07, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, "sambanova/Meta-Llama-3.3-70B-Instruct": { "input_cost_per_token": 6e-07, "litellm_provider": "sambanova", @@ -46625,54 +43081,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "sambanova/Meta-Llama-Guard-3-8B": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 3e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, - "sambanova/QwQ-32B": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 5e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-06, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, - "sambanova/Qwen2-Audio-7B-Instruct": { - "deprecation_date": "2025-06-19", - "input_cost_per_token": 5e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 0.0001, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_audio_input": true - }, - "sambanova/Qwen3-32B": { - "deprecation_date": "2026-04-06", - "input_cost_per_token": 4e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "sambanova/DeepSeek-V3.1": { "max_tokens": 131072, "max_input_tokens": 131072, @@ -47265,27 +43673,6 @@ "mode": "image_generation", "output_cost_per_image": 0.14 }, - "standard/1024-x-1024/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 3.81469e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, - "standard/1024-x-1792/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 4.359e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, - "standard/1792-x-1024/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 4.359e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "linkup/search": { "input_cost_per_query": 0.00587, "litellm_provider": "linkup", @@ -47420,36 +43807,6 @@ "output_vector_size": 768, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, - "text-moderation-007": { - "deprecation_date": "2025-10-27", - "input_cost_per_token": 0.0, - "litellm_provider": "openai", - "max_input_tokens": 32768, - "max_output_tokens": 0, - "max_tokens": 0, - "mode": "moderation", - "output_cost_per_token": 0.0 - }, - "text-moderation-latest": { - "deprecation_date": "2025-10-27", - "input_cost_per_token": 0.0, - "litellm_provider": "openai", - "max_input_tokens": 32768, - "max_output_tokens": 0, - "max_tokens": 0, - "mode": "moderation", - "output_cost_per_token": 0.0 - }, - "text-moderation-stable": { - "deprecation_date": "2025-10-27", - "input_cost_per_token": 0.0, - "litellm_provider": "openai", - "max_input_tokens": 32768, - "max_output_tokens": 0, - "max_tokens": 0, - "mode": "moderation", - "output_cost_per_token": 0.0 - }, "text-multilingual-embedding-002": { "deprecation_date": "2027-04-01", "input_cost_per_character": 2.5e-08, @@ -47547,19 +43904,6 @@ "mode": "chat", "output_cost_per_token": 1e-07 }, - "together_ai/Qwen/Qwen2.5-72B-Instruct-Turbo": { - "deprecation_date": "2026-02-06", - "litellm_provider": "together_ai", - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "input_cost_per_token": 1.2e-06, - "output_cost_per_token": 1.2e-06, - "max_input_tokens": 131072, - "source": "https://api.together.ai/v1/models" - }, "together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo": { "litellm_provider": "together_ai", "mode": "chat", @@ -47572,87 +43916,6 @@ "max_input_tokens": 32768, "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { - "deprecation_date": "2026-07-10", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262000, - "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { - "deprecation_date": "2026-04-16", - "input_cost_per_token": 6.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": { - "deprecation_date": "2026-02-06", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://www.together.ai/models/qwen3-235b-a22b-fp8-tput", - "supports_function_calling": false, - "supports_parallel_function_calling": false, - "supports_tool_choice": false - }, - "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { - "deprecation_date": "2026-06-04", - "input_cost_per_token": 2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/deepseek-ai/DeepSeek-R1": { - "deprecation_date": "2026-05-14", - "input_cost_per_token": 3e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 128000, - "max_output_tokens": 20480, - "max_tokens": 20480, - "metadata": { - "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" - }, - "mode": "chat", - "output_cost_per_token": 7e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/deepseek-ai/DeepSeek-R1-0528-tput": { - "deprecation_date": "2026-02-03", - "input_cost_per_token": 5.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.19e-06, - "source": "https://www.together.ai/models/deepseek-r1-0528-throughput", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/deepseek-ai/DeepSeek-V3": { "input_cost_per_token": 1.25e-06, "litellm_provider": "together_ai", @@ -47669,33 +43932,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/deepseek-ai/DeepSeek-V3.1": { - "deprecation_date": "2026-05-14", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "max_tokens": 16384, - "metadata": { - "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" - }, - "mode": "chat", - "output_cost_per_token": 1.7e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "max_input_tokens": 131072, - "max_output_tokens": 16384 - }, - "together_ai/meta-llama/Llama-3.2-3B-Instruct-Turbo": { - "deprecation_date": "2026-03-06", - "litellm_provider": "together_ai", - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "input_cost_per_token": 1.04e-06, "litellm_provider": "together_ai", @@ -47709,112 +43945,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo-Free": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 0, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 0, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { - "deprecation_date": "2026-03-31", - "input_cost_per_token": 2.7e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8.5e-07, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Llama-4-Scout-17B-16E-Instruct": { - "deprecation_date": "2026-02-06", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 5.9e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo": { - "deprecation_date": "2026-02-06", - "input_cost_per_token": 3.5e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 3.5e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { - "deprecation_date": "2026-02-25", - "input_cost_per_token": 8.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8.8e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { - "deprecation_date": "2026-03-06", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/mistralai/Mistral-7B-Instruct-v0.1": { - "deprecation_date": "2025-11-13", - "litellm_provider": "together_ai", - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "input_cost_per_token": 2e-07, - "output_cost_per_token": 2e-07, - "max_input_tokens": 32768, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": { - "deprecation_date": "2026-04-02", - "litellm_provider": "together_ai", - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 3e-07, - "max_input_tokens": 32768, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1": { - "deprecation_date": "2026-04-16", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/moonshotai/Kimi-K2-Instruct": { "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", @@ -47843,19 +43973,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/openai/gpt-oss-20b": { - "deprecation_date": "2026-09-14", - "input_cost_per_token": 5e-08, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/togethercomputer/CodeLlama-34b-Instruct": { "litellm_provider": "together_ai", "mode": "chat", @@ -47863,19 +43980,6 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true }, - "together_ai/zai-org/GLM-4.5-Air-FP8": { - "deprecation_date": "2026-04-02", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.1e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/zai-org/GLM-4.6": { "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", @@ -47892,102 +43996,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "together_ai/zai-org/GLM-4.7": { - "deprecation_date": "2026-04-02", - "input_cost_per_token": 4.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 202752, - "max_tokens": 202752, - "metadata": { - "successor": "together_ai/zai-org/GLM-5.2" - }, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, - "together_ai/moonshotai/Kimi-K2.5": { - "deprecation_date": "2026-05-21", - "input_cost_per_token": 5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 256000, - "max_tokens": 256000, - "metadata": { - "successor": "together_ai/moonshotai/Kimi-K3" - }, - "mode": "chat", - "output_cost_per_token": 2.8e-06, - "source": "https://www.together.ai/models/kimi-k2-5", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_reasoning": true - }, - "together_ai/moonshotai/Kimi-K2-Instruct-0905": { - "deprecation_date": "2026-03-06", - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "metadata": { - "successor": "together_ai/moonshotai/Kimi-K3" - }, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://www.together.ai/models/kimi-k2-0905", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3-Next-80B-A3B-Instruct": { - "deprecation_date": "2026-04-02", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "metadata": { - "successor": "together_ai/Qwen/Qwen3.7-Plus" - }, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": { - "deprecation_date": "2026-02-25", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "metadata": { - "successor": "together_ai/Qwen/Qwen3.6-Plus" - }, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3.5-397B-A17B": { - "cache_read_input_token_cost": 3.5e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 3.6e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/MiniMaxAI/MiniMax-M3": { "cache_read_input_token_cost": 6e-08, "input_cost_per_token": 3e-07, @@ -48109,23 +44117,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/deepseek-ai/DeepSeek-V4-Pro": { - "deprecation_date": "2026-08-27", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.74e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 512000, - "max_tokens": 512000, - "mode": "chat", - "output_cost_per_token": 3.48e-06, - "source": "https://docs.together.ai/docs/serverless-models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": { "cache_read_input_token_cost": 1.3e-07, "deprecation_date": "2026-09-29", @@ -48142,52 +44133,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/google/gemma-3n-E4B-it": { - "deprecation_date": "2026-08-25", - "input_cost_per_token": 6e-08, - "litellm_provider": "together_ai", - "max_input_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1.2e-07, - "source": "https://docs.together.ai/docs/serverless-models" - }, - "together_ai/google/gemma-4-31B-it": { - "deprecation_date": "2026-09-14", - "input_cost_per_token": 3.9e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 9.7e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "together_ai/intfloat/multilingual-e5-large-instruct": { - "deprecation_date": "2026-09-14", - "input_cost_per_token": 2e-08, - "litellm_provider": "together_ai", - "max_input_tokens": 514, - "max_tokens": 514, - "mode": "embedding", - "output_cost_per_token": 2e-08, - "output_vector_size": 1024, - "source": "https://docs.together.ai/docs/serverless-models" - }, - "together_ai/meta-llama/Llama-Guard-4-12B": { - "deprecation_date": "2026-08-25", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 1048576, - "max_tokens": 1048576, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://docs.together.ai/docs/serverless-models" - }, "together_ai/meta-models/Muse-Glimmer-30B": { "cache_read_input_token_cost": 4e-08, "input_cost_per_token": 3.5e-07, @@ -48199,23 +44144,6 @@ "source": "https://api.together.ai/v1/models", "supports_prompt_caching": true }, - "together_ai/moonshotai/Kimi-K2.7-Code": { - "deprecation_date": "2026-08-27", - "cache_read_input_token_cost": 1.9e-07, - "input_cost_per_token": 9.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 4e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "together_ai/moonshotai/Kimi-K3": { "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, @@ -48238,33 +44166,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "together_ai/nvidia/nemotron-3-ultra-550b-a55b": { - "deprecation_date": "2026-08-27", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 512288, - "max_tokens": 512288, - "mode": "chat", - "output_cost_per_token": 3.6e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/pearl-ai/gemma-4-31b-it": { - "deprecation_date": "2026-08-27", - "input_cost_per_token": 2.8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 8.6e-07, - "source": "https://docs.together.ai/docs/serverless-models" - }, "together_ai/thinkingmachines/Inkling": { "cache_read_input_token_cost": 1.7e-07, "input_cost_per_token": 1e-06, @@ -48280,18 +44181,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/thinkingmachines/Inkling-Small": { - "deprecation_date": "2026-09-14", - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 524288, - "max_tokens": 524288, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://api.together.ai/v1/models", - "supports_prompt_caching": true - }, "together_ai/zai-org/GLM-5.2": { "cache_read_input_token_cost": 2.6e-07, "input_cost_per_token": 1.4e-06, @@ -48438,22 +44327,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "us.amazon.nova-premier-v1:0": { - "deprecation_date": "2026-09-14", - "input_cost_per_token": 2.5e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 10000, - "max_tokens": 10000, - "mode": "chat", - "output_cost_per_token": 1.25e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_vision": true, - "cache_read_input_token_cost": 6.25e-07 - }, "us.amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 8e-07, @@ -48571,23 +44444,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "us.anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 - }, "us.anthropic.claude-3-opus-20240229-v1:0": { "input_cost_per_token": 1.5e-05, "litellm_provider": "bedrock", @@ -48603,23 +44459,6 @@ "cache_read_input_token_cost": 1.5e-06, "cache_creation_input_token_cost": 1.875e-05 }, - "us.anthropic.claude-3-sonnet-20240229-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, "us.anthropic.claude-opus-4-1-20250805-v1:0": { "cache_creation_input_token_cost": 1.875e-05, "cache_read_input_token_cost": 1.5e-06, @@ -48684,23 +44523,6 @@ "input_cost_per_token_batches": 1.65e-06, "output_cost_per_token_batches": 8.25e-06 }, - "us-gov.anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 3e-07, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-08, - "cache_creation_input_token_cost": 3.75e-07 - }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, @@ -48832,6 +44654,41 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "us-gov.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 6e-06, + "cache_creation_input_token_cost_above_1hr": 9.6e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 4.8e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.4e-05, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "thinking_always_on": true, + "supports_forced_tool_use": false + }, "us-gov.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.5e-05, "cache_creation_input_token_cost_above_1hr": 2.4e-05, @@ -49828,34 +45685,6 @@ "output_cost_per_token": 9e-07, "supports_tool_choice": true }, - "vercel_ai_gateway/google/gemini-2.0-flash": { - "deprecation_date": "2026-06-01", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "vercel_ai_gateway", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_vision": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true - }, - "vercel_ai_gateway/google/gemini-2.0-flash-lite": { - "deprecation_date": "2026-06-01", - "input_cost_per_token": 7.5e-08, - "litellm_provider": "vercel_ai_gateway", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "supports_vision": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true - }, "vercel_ai_gateway/google/gemini-2.5-flash": { "input_cost_per_token": 3e-07, "litellm_provider": "vercel_ai_gateway", @@ -50739,28 +46568,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "vertex_ai/claude-3-7-sonnet@20250219": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-05-11", - "input_cost_per_token": 3e-06, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "vertex_ai/claude-3-haiku": { "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-anthropic_models", @@ -50839,72 +46646,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "vertex_ai/claude-opus-4": { - "deprecation_date": "2026-05-14", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "vertex_ai/claude-opus-4-1": { - "deprecation_date": "2026-08-05", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "input_cost_per_token_batches": 7.5e-06, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "output_cost_per_token_batches": 3.75e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "vertex_ai/claude-opus-4-1@20250805": { - "deprecation_date": "2026-08-05", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "input_cost_per_token_batches": 7.5e-06, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "output_cost_per_token_batches": 3.75e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "vertex_ai/claude-opus-4-5": { "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, @@ -51310,6 +47051,78 @@ "supports_max_reasoning_effort": true, "prompt_cache_min_tokens": 512 }, + "vertex_ai/claude-opus-5-5": { + "regional_endpoint_uplift_multiplier": 1.1, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 + }, + "vertex_ai/claude-opus-5-5@default": { + "regional_endpoint_uplift_multiplier": 1.1, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 + }, "vertex_ai/claude-opus-4-8": { "deprecation_date": "2027-05-28", "regional_endpoint_uplift_multiplier": 1.1, @@ -51505,98 +47318,6 @@ "supports_native_streaming": true, "prompt_cache_min_tokens": 1024 }, - "vertex_ai/claude-opus-4@20250514": { - "deprecation_date": "2026-05-14", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "vertex_ai/claude-sonnet-4": { - "deprecation_date": "2026-05-14", - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 1000000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "vertex_ai/claude-sonnet-4@20250514": { - "deprecation_date": "2026-05-14", - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 1000000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, "vertex_ai/mistralai/codestral-2@001": { "input_cost_per_token": 3e-07, "litellm_provider": "vertex_ai-mistral_models", @@ -51785,6 +47506,7 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -51809,7 +47531,8 @@ "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05, "supports_reasoning": false, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "cache_read_input_token_cost_batches": 1e-07 }, "vertex_ai/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -51829,6 +47552,7 @@ }, "vertex_ai/gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -51846,7 +47570,8 @@ "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, "supports_reasoning": false, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "cache_read_input_token_cost_batches": 2.5e-08 }, "vertex_ai/gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, @@ -51864,6 +47589,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -51901,7 +47627,8 @@ "supports_response_schema": false, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.25e-08 }, "vertex_ai/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, @@ -51956,6 +47683,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -52011,11 +47739,13 @@ }, "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_audio_token_batches": 2.5e-07 + "input_cost_per_audio_token_batches": 2.5e-07, + "cache_read_input_token_cost_batches": 1.25e-08 }, "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -52069,10 +47799,12 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 1.5e-08 }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -52085,63 +47817,8 @@ "output_cost_per_image_token": 0.00012, "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" - }, - "vertex_ai/imagegeneration@006": { - "deprecation_date": "2025-09-24", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-3.0-fast-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-3.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-3.0-generate-002": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-3.0-capability-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/image/edit-insert-objects" - }, - "vertex_ai/imagen-4.0-fast-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-4.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-4.0-ultra-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "cache_read_input_token_cost_batches": 1e-07 }, "vertex_ai/jamba-1.5": { "input_cost_per_token": 2e-07, @@ -52887,51 +48564,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "vertex_ai/veo-2.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-video-models", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "video_generation", - "output_cost_per_second": 0.35, - "source": "https://ai.google.dev/gemini-api/docs/video", - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "video" - ] - }, - "vertex_ai/veo-3.0-fast-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-video-models", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "video_generation", - "output_cost_per_second": 0.15, - "source": "https://ai.google.dev/gemini-api/docs/video", - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "video" - ] - }, - "vertex_ai/veo-3.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-video-models", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "video_generation", - "output_cost_per_second": 0.4, - "source": "https://ai.google.dev/gemini-api/docs/video", - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "video" - ] - }, "vertex_ai/veo-3.1-generate-preview": { "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, @@ -53220,59 +48852,6 @@ "mode": "chat", "source": "https://wandb.ai/site/pricing/tokens/" }, - "wandb/zai-org/GLM-4.5": { - "deprecation_date": "2026-03-04", - "supports_reasoning": true, - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 0.055, - "output_cost_per_token": 0.2, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/Qwen/Qwen3-235B-A22B-Instruct-2507": { - "deprecation_date": "2026-08-04", - "max_tokens": 262144, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 1e-07, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/Qwen/Qwen3-Coder-480B-A35B-Instruct": { - "deprecation_date": "2026-08-25", - "max_tokens": 262144, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 1e-06, - "output_cost_per_token": 1.5e-06, - "litellm_provider": "wandb", - "mode": "chat", - "source": "https://wandb.ai/site/pricing/tokens/" - }, - "wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": { - "deprecation_date": "2026-08-04", - "supports_reasoning": true, - "max_tokens": 262144, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 1e-07, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/moonshotai/Kimi-K2-Instruct": { - "deprecation_date": "2026-03-04", - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 6e-07, - "output_cost_per_token": 2.5e-06, - "litellm_provider": "wandb", - "mode": "chat" - }, "wandb/moonshotai/Kimi-K2.5": { "max_tokens": 262144, "max_input_tokens": 262144, @@ -53288,20 +48867,6 @@ "supports_response_schema": true, "supports_vision": true }, - "wandb/MiniMaxAI/MiniMax-M2.5": { - "deprecation_date": "2026-08-25", - "max_tokens": 197000, - "max_input_tokens": 197000, - "max_output_tokens": 197000, - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "wandb", - "mode": "chat", - "source": "https://wandb.ai/inference/coreweave/cw_MiniMaxAI_MiniMax-M2.5", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true - }, "wandb/meta-llama/Llama-3.1-8B-Instruct": { "max_tokens": 128000, "max_input_tokens": 131000, @@ -53323,27 +48888,6 @@ "mode": "chat", "source": "https://wandb.ai/site/pricing/tokens/" }, - "wandb/deepseek-ai/DeepSeek-R1-0528": { - "deprecation_date": "2026-03-04", - "supports_reasoning": true, - "max_tokens": 161000, - "max_input_tokens": 161000, - "max_output_tokens": 161000, - "input_cost_per_token": 1.35e-06, - "output_cost_per_token": 5.4e-06, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/deepseek-ai/DeepSeek-V3-0324": { - "deprecation_date": "2026-03-04", - "max_tokens": 161000, - "max_input_tokens": 161000, - "max_output_tokens": 161000, - "input_cost_per_token": 1.14e-06, - "output_cost_per_token": 2.75e-06, - "litellm_provider": "wandb", - "mode": "chat" - }, "wandb/meta-llama/Llama-3.3-70B-Instruct": { "max_tokens": 128000, "max_input_tokens": 128000, @@ -53354,26 +48898,6 @@ "mode": "chat", "source": "https://wandb.ai/site/pricing/tokens/" }, - "wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": { - "deprecation_date": "2026-04-21", - "max_tokens": 64000, - "max_input_tokens": 64000, - "max_output_tokens": 64000, - "input_cost_per_token": 1.7e-07, - "output_cost_per_token": 6.6e-07, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/microsoft/Phi-4-mini-instruct": { - "deprecation_date": "2026-08-04", - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 0.008, - "output_cost_per_token": 0.035, - "litellm_provider": "wandb", - "mode": "chat" - }, "watsonx/ibm/granite-3-8b-instruct": { "input_cost_per_token": 2e-07, "litellm_provider": "watsonx", @@ -53774,440 +49298,6 @@ "deprecation_date": "2027-02-26", "source": "https://developers.openai.com/api/docs/pricing" }, - "xai/grok-3": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-beta": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-fast-beta": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-fast-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini": { - "cache_read_input_token_cost": 2e-07, - "deprecation_date": "2026-02-28", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-beta": { - "cache_read_input_token_cost": 2e-07, - "deprecation_date": "2026-02-28", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-fast": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-02-28", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-fast-beta": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-02-28", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-fast-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-02-28", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-02-28", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4": { - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-fast-reasoning": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-fast-non-reasoning": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-0709": { - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-latest": { - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast-reasoning": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast-reasoning-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast-non-reasoning": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast-non-reasoning-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, "xai/grok-4.20-multi-agent-beta-0309": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.25e-06, @@ -54217,7 +49307,7 @@ "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54227,6 +49317,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true, "supported_endpoints": [ "/v1/responses" @@ -54241,7 +49332,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54251,6 +49342,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-0309-reasoning": { @@ -54262,7 +49354,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -54271,6 +49363,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, @@ -54283,7 +49376,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, @@ -54292,11 +49385,13 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.3": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "xai", @@ -54306,7 +49401,7 @@ "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54318,6 +49413,7 @@ "xai/grok-4.3-latest": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "xai", @@ -54327,7 +49423,7 @@ "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54339,6 +49435,7 @@ "xai/grok-4.5": { "cache_read_input_token_cost": 3e-07, "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54348,7 +49445,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54360,6 +49457,7 @@ "xai/grok-4.5-latest": { "cache_read_input_token_cost": 3e-07, "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54369,7 +49467,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54381,6 +49479,7 @@ "xai/grok-build-latest": { "cache_read_input_token_cost": 3e-07, "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54390,7 +49489,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/developers/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54402,6 +49501,7 @@ "xai/grok-4.6": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54411,7 +49511,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/developers/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54423,6 +49523,7 @@ "xai/grok-4.7": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54432,7 +49533,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/developers/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54441,69 +49542,6 @@ "supports_vision": true, "supports_web_search": true }, - "xai/grok-code-fast": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "input_cost_per_token_above_200k_tokens": 2e-06, - "output_cost_per_token_above_200k_tokens": 4e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "supports_response_schema": true, - "supports_vision": true, - "deprecation_date": "2026-05-15" - }, - "xai/grok-code-fast-1": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "input_cost_per_token_above_200k_tokens": 2e-06, - "output_cost_per_token_above_200k_tokens": 4e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "supports_response_schema": true, - "supports_vision": true, - "deprecation_date": "2026-05-15" - }, - "xai/grok-code-fast-1-0825": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "input_cost_per_token_above_200k_tokens": 2e-06, - "output_cost_per_token_above_200k_tokens": 4e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "supports_response_schema": true, - "supports_vision": true, - "deprecation_date": "2026-05-15" - }, "zai.glm-4.7": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", @@ -57246,30 +52284,6 @@ "supports_reasoning": true, "supports_vision": true }, - "scaleway/google/gemma-3-27b-it": { - "input_cost_per_token": 2.5e-07, - "litellm_provider": "scaleway", - "max_input_tokens": 40000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_function_calling": true, - "supports_vision": true, - "deprecation_date": "2026-08-01" - }, - "scaleway/hcompany/holo2-30b-a3b": { - "input_cost_per_token": 3e-07, - "litellm_provider": "scaleway", - "max_input_tokens": 22000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 7e-07, - "supports_reasoning": true, - "supports_vision": true, - "deprecation_date": "2026-08-09" - }, "scaleway/mistralai/mistral-medium-3.5-128b": { "input_cost_per_token": 1.5e-06, "litellm_provider": "scaleway", @@ -57283,29 +52297,6 @@ "supports_vision": true, "supports_tool_choice": true }, - "scaleway/mistralai/devstral-2-123b-instruct-2512": { - "input_cost_per_token": 4e-07, - "litellm_provider": "scaleway", - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 2e-06, - "supports_function_calling": true, - "deprecation_date": "2026-08-01" - }, - "scaleway/mistralai/voxtral-small-24b-2507": { - "input_cost_per_audio_token": 1.5e-07, - "input_cost_per_token": 1.5e-07, - "litellm_provider": "scaleway", - "max_input_tokens": 32000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 3.5e-07, - "supports_audio_input": true, - "deprecation_date": "2026-08-01" - }, "scaleway/mistralai/mistral-small-3.2-24b-instruct-2506": { "input_cost_per_token": 1.5e-07, "litellm_provider": "scaleway", @@ -58865,26 +53856,6 @@ "/v1/audio/speech" ] }, - "gpt-4o-mini-tts-2025-03-20": { - "deprecation_date": "2026-07-23", - "input_cost_per_token": 6e-07, - "litellm_provider": "openai", - "mode": "audio_speech", - "output_cost_per_audio_token": 1.2e-05, - "output_cost_per_second": 0.00025, - "output_cost_per_token": 1e-05, - "source": "https://developers.openai.com/api/docs/pricing", - "supported_endpoints": [ - "/v1/audio/speech" - ], - "supported_modalities": [ - "text", - "audio" - ], - "supported_output_modalities": [ - "audio" - ] - }, "gpt-4o-mini-tts-2025-12-15": { "input_cost_per_token": 6e-07, "litellm_provider": "openai", @@ -58988,41 +53959,6 @@ "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false }, - "gpt-realtime-mini-2025-10-06": { - "cache_creation_input_audio_token_cost": 3e-07, - "cache_read_input_audio_token_cost": 3e-07, - "cache_read_input_token_cost": 6e-08, - "deprecation_date": "2026-07-23", - "input_cost_per_audio_token": 1e-05, - "input_cost_per_image_token": 8e-07, - "input_cost_per_token": 6e-07, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 2.4e-06, - "source": "https://developers.openai.com/api/docs/pricing", - "supported_endpoints": [ - "/v1/realtime" - ], - "supported_modalities": [ - "text", - "image", - "audio" - ], - "supported_output_modalities": [ - "text", - "audio" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-realtime-mini-2025-12-15": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, @@ -59118,6 +54054,7 @@ }, "chatgpt-image-latest": { "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", "input_cost_per_image_token": 8e-06, "input_cost_per_image_token_batches": 4e-06, @@ -59176,42 +54113,6 @@ "tpm": 250000, "rpm": 10 }, - "gemini/gemini-2.0-flash-lite-001": { - "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7.5e-08, - "input_cost_per_token": 7.5e-08, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "rpm": 4000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.0-flash-lite", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 4000000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, "gemini-2.5-flash-native-audio-latest": { "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, @@ -61978,7 +56879,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, @@ -61987,6 +56888,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent-0309": { @@ -61998,7 +56900,7 @@ "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": false, "supports_prompt_caching": true, "supports_reasoning": true, @@ -62008,6 +56910,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true, "supported_endpoints": [ "/v1/responses" @@ -62022,7 +56925,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -62030,6 +56933,7 @@ "input_cost_per_token_above_200k_tokens": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, "supports_response_schema": true, "supports_vision": true }, @@ -62105,6 +57009,7 @@ "supports_audio_output": true }, "claude-mythos-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -62144,6 +57049,7 @@ } }, "claude-mythos-5-1": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 2.5e-07, @@ -62184,6 +57090,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-mythos-preview": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -65225,7 +60132,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -65234,6 +60141,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, @@ -65246,7 +60154,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -65255,6 +60163,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, @@ -65267,7 +60176,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -65276,6 +60185,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, @@ -65362,23 +60272,6 @@ "image" ] }, - "xai/grok-imagine-image-pro": { - "input_cost_per_image": 0.05, - "litellm_provider": "xai", - "mode": "image_generation", - "source": "https://docs.x.ai/docs/models", - "supported_endpoints": [ - "/v1/images/generations" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "image" - ], - "deprecation_date": "2026-05-15" - }, "xai/grok-imagine-image-2.0": { "input_cost_per_image": 0.06, "litellm_provider": "xai", @@ -65519,7 +60412,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, @@ -65528,6 +60421,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-non-reasoning-latest": { @@ -65539,7 +60433,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, @@ -65548,6 +60442,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent": { @@ -65559,7 +60454,7 @@ "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supported_endpoints": [ "/v1/responses" ], @@ -65572,6 +60467,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent-latest": { @@ -65583,7 +60479,7 @@ "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supported_endpoints": [ "/v1/responses" ], @@ -65596,6 +60492,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "groq/qwen/qwen3.8-27b": { @@ -65992,16 +60889,6 @@ "output_cost_per_token": 2.82e-07, "source": "https://api.together.ai/v1/models" }, - "together_ai/moonshotai/Kimi-K2.6": { - "deprecation_date": "2026-08-19", - "input_cost_per_token": 1.2e-06, - "output_cost_per_token": 4.5e-06, - "cache_read_input_token_cost": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, "together_ai/moonshotai/Kimi-K2.5-fp4": { "input_cost_per_token": 5e-07, "output_cost_per_token": 2.8e-06, @@ -66019,25 +60906,6 @@ "mode": "chat", "source": "https://api.together.ai/v1/models" }, - "together_ai/zai-org/GLM-5": { - "deprecation_date": "2026-06-22", - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3.2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 202752, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, - "together_ai/zai-org/GLM-5.1": { - "deprecation_date": "2026-07-10", - "input_cost_per_token": 1.4e-06, - "output_cost_per_token": 4.4e-06, - "cache_read_input_token_cost": 2.6e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 202752, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, "together_ai/deepseek-ai/DeepSeek-R1-0528": { "input_cost_per_token": 3e-06, "output_cost_per_token": 7e-06, @@ -66046,33 +60914,6 @@ "mode": "chat", "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen3-Coder-Next-FP8": { - "deprecation_date": "2026-05-14", - "input_cost_per_token": 5e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen3-VL-32B-Instruct": { - "deprecation_date": "2026-02-25", - "input_cost_per_token": 5e-07, - "output_cost_per_token": 1.5e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen3-VL-8B-Instruct": { - "deprecation_date": "2026-04-16", - "input_cost_per_token": 1.8e-07, - "output_cost_per_token": 6.8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, "together_ai/mistralai/Ministral-3-14B-Instruct-2512": { "input_cost_per_token": 2e-07, "output_cost_per_token": 2e-07, @@ -66097,15 +60938,6 @@ "mode": "chat", "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/QwQ-32B": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 1.2e-06, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, "cerebras/gemma-4-31b": { "input_cost_per_token": 9.9e-07, "litellm_provider": "cerebras", @@ -66357,6 +61189,41 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "bedrock/us-gov-west-1/anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 6e-06, + "cache_creation_input_token_cost_above_1hr": 9.6e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 4.8e-06, + "litellm_provider": "bedrock", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.4e-05, + "prompt_cache_min_tokens": 512, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-gov-west-1/anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.5e-05, "cache_creation_input_token_cost_above_1hr": 2.4e-05, @@ -66563,6 +61430,41 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "bedrock/us-gov-east-1/anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 6e-06, + "cache_creation_input_token_cost_above_1hr": 9.6e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 4.8e-06, + "litellm_provider": "bedrock", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.4e-05, + "prompt_cache_min_tokens": 512, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-gov-east-1/anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.5e-05, "cache_creation_input_token_cost_above_1hr": 2.4e-05, @@ -68085,7 +62987,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://www.baseten.co/pricing/", + "source": "https://inference.baseten.co/v1/models", "supported_modalities": [ "text", "image" @@ -68095,6 +62997,7 @@ ], "supports_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true @@ -68252,9 +63155,9 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3": { - "input_cost_per_token": 9.1e-07, - "output_cost_per_token": 2.86e-06, - "cache_read_input_token_cost": 1.69e-07, + "input_cost_per_token": 6.538e-07, + "output_cost_per_token": 2.0548e-06, + "cache_read_input_token_cost": 1.2142e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 131072, @@ -68620,7 +63523,7 @@ }, "openrouter/moonshotai/kimi-k2.7-code": { "input_cost_per_token": 7.062e-07, - "output_cost_per_token": 3.21e-06, + "output_cost_per_token": 3.3e-06, "cache_read_input_token_cost": 1.8e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, @@ -68880,13 +63783,13 @@ "supports_web_search": false }, "openrouter/qwen/qwen3.6-27b": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 2e-06, - "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3.2e-07, + "output_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_output_tokens": 262140, + "max_tokens": 262140, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -68941,9 +63844,9 @@ "supports_web_search": true }, "openrouter/deepseek/deepseek-v4-flash": { - "input_cost_per_token": 8.8606e-08, - "output_cost_per_token": 1.77212e-07, - "cache_read_input_token_cost": 1.77212e-08, + "input_cost_per_token": 4.9e-08, + "output_cost_per_token": 9.8e-08, + "cache_read_input_token_cost": 9.8e-09, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, @@ -69803,8 +64706,8 @@ "output_cost_per_token": 1.2e-06, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 235929, + "max_tokens": 235929, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -70847,38 +65750,6 @@ "output_cost_per_token": 1.5e-07, "source": "https://api.together.ai/v1/models" }, - "together_ai/deepseek-ai/deepseek-coder-33b-instruct": { - "deprecation_date": "2024-08-22", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { - "deprecation_date": "2025-12-23", - "input_cost_per_token": 2e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 1.6e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1.6e-06, - "source": "https://api.together.ai/v1/models" - }, "vertex_ai/gemini-2.5-flash-native-audio": { "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, @@ -70897,16 +65768,6 @@ "output_cost_per_token": 1e-05, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, - "vertex_ai/gemini-3.1-flash-live-preview": { - "input_cost_per_audio_token": 3e-06, - "input_cost_per_second": 8.33333333333e-05, - "input_cost_per_token": 7.5e-07, - "litellm_provider": "vertex_ai", - "mode": "realtime", - "output_cost_per_audio_token": 1.2e-05, - "output_cost_per_token": 4.5e-06, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" - }, "vertex_ai/gemini-3.1-flash-tts-preview": { "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, @@ -70939,16 +65800,6 @@ "output_cost_per_token": 9e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, - "vertex_ai/gemini-robotics-er-2": { - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, - "litellm_provider": "vertex_ai", - "mode": "chat", - "output_cost_per_token": 5e-06, - "output_cost_per_token_batches": 2.5e-06, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" - }, "vertex_ai/gemma-4-26b-a4b-it": { "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, @@ -70957,14 +65808,6 @@ "output_cost_per_token": 6e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, - "together_ai/google/gemma-2-27b-it": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, "gpt-5.5-cyber": { "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 1.25e-05, @@ -70982,14 +65825,6 @@ "output_cost_per_token": 2.5e-05, "source": "https://developers.openai.com/api/docs/pricing" }, - "together_ai/meta-llama/Llama-3-8b-chat-hf": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/meta-llama/Llama-3.1-405B-Instruct": { "input_cost_per_token": 3.5e-06, "litellm_provider": "together_ai", @@ -71011,38 +65846,6 @@ "output_cost_per_token": 6e-08, "source": "https://api.together.ai/v1/models" }, - "together_ai/meta-llama/Meta-Llama-3-70B-Instruct-Turbo": { - "deprecation_date": "2025-12-23", - "input_cost_per_token": 8.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8.8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/meta-llama/Meta-Llama-3-8B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 8.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8.8e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/Qwen/Qwen2-1.5B-Instruct": { "input_cost_per_token": 2e-08, "litellm_provider": "together_ai", @@ -71050,22 +65853,6 @@ "output_cost_per_token": 2e-08, "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen2-72B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 9e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 9e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen2-VL-72B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 1.2e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://api.together.ai/v1/models" - }, "together_ai/Qwen/Qwen2.5-14B-Instruct": { "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", @@ -71080,22 +65867,6 @@ "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen2.5-Coder-32B-Instruct": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen2.5-VL-72B-Instruct": { - "deprecation_date": "2026-01-05", - "input_cost_per_token": 1.95e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8e-06, - "source": "https://api.together.ai/v1/models" - }, "azure/eu/codex-mini": { "deprecation_date": "2026-11-15", "cache_read_input_token_cost": 4.13e-07, @@ -71164,6 +65935,7 @@ "azure/eu/gpt-5": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_batches": 6.875e-08, "cache_read_input_token_cost_priority": 2.75e-07, "input_cost_per_token": 1.375e-06, "input_cost_per_token_batches": 6.875e-07, @@ -71187,6 +65959,7 @@ "azure/eu/gpt-5-mini": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 2.75e-08, + "cache_read_input_token_cost_batches": 1.375e-08, "cache_read_input_token_cost_priority": 4.95e-08, "input_cost_per_token": 2.75e-07, "input_cost_per_token_batches": 1.375e-07, @@ -71201,6 +65974,7 @@ "azure/eu/gpt-5-nano": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 5.5e-09, + "cache_read_input_token_cost_batches": 2.75e-09, "input_cost_per_token": 5.5e-08, "input_cost_per_token_batches": 2.75e-08, "litellm_provider": "azure", @@ -71231,6 +66005,7 @@ "azure/eu/gpt-5.2": { "deprecation_date": "2027-06-08", "cache_read_input_token_cost": 1.925e-07, + "cache_read_input_token_cost_batches": 9.625e-08, "cache_read_input_token_cost_priority": 3.85e-07, "input_cost_per_token": 1.925e-06, "input_cost_per_token_batches": 9.625e-07, @@ -71242,15 +66017,6 @@ "output_cost_per_token_priority": 3.08e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/eu/gpt-5.2-chat": { - "deprecation_date": "2026-06-29", - "cache_read_input_token_cost": 1.925e-07, - "input_cost_per_token": 1.925e-06, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 1.54e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/eu/gpt-5.2-codex": { "deprecation_date": "2027-07-13", "cache_read_input_token_cost": 1.925e-07, @@ -71269,15 +66035,6 @@ "output_cost_per_token_batches": 9.24e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/eu/gpt-5.3-chat": { - "deprecation_date": "2026-06-29", - "cache_read_input_token_cost": 1.925e-07, - "input_cost_per_token": 1.925e-06, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 1.54e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/eu/gpt-5.3-codex": { "deprecation_date": "2027-08-24", "cache_read_input_token_cost": 1.925e-07, @@ -71293,6 +66050,7 @@ "azure/eu/gpt-5.4-mini": { "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 8.25e-08, + "cache_read_input_token_cost_batches": 4.125e-08, "cache_read_input_token_cost_priority": 1.65e-07, "input_cost_per_token": 8.25e-07, "input_cost_per_token_batches": 4.125e-07, @@ -71307,6 +66065,7 @@ "azure/eu/gpt-5.4-nano": { "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2.2e-08, + "cache_read_input_token_cost_batches": 1.1e-08, "input_cost_per_token": 2.2e-07, "input_cost_per_token_batches": 1.1e-07, "litellm_provider": "azure", @@ -71319,11 +66078,13 @@ "deprecation_date": "2027-09-07", "input_cost_per_token": 3.3e-05, "input_cost_per_token_above_272k_tokens": 6.6e-05, + "input_cost_per_token_above_272k_tokens_batches": 3.3e-05, "input_cost_per_token_batches": 1.65e-05, "litellm_provider": "azure", "mode": "chat", "output_cost_per_token": 0.000198, "output_cost_per_token_above_272k_tokens": 0.000297, + "output_cost_per_token_above_272k_tokens_batches": 0.0001485, "output_cost_per_token_batches": 9.9e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, @@ -71350,14 +66111,6 @@ "output_cost_per_token_batches": 2.42e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/eu/o1-preview": { - "cache_read_input_token_cost": 8.25e-06, - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/eu/o3-2025-04-16": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, @@ -71518,6 +66271,7 @@ "azure/us/gpt-5": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_batches": 6.875e-08, "cache_read_input_token_cost_priority": 2.75e-07, "input_cost_per_token": 1.375e-06, "input_cost_per_token_batches": 6.875e-07, @@ -71541,6 +66295,7 @@ "azure/us/gpt-5-mini": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 2.75e-08, + "cache_read_input_token_cost_batches": 1.375e-08, "cache_read_input_token_cost_priority": 4.95e-08, "input_cost_per_token": 2.75e-07, "input_cost_per_token_batches": 1.375e-07, @@ -71555,6 +66310,7 @@ "azure/us/gpt-5-nano": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 5.5e-09, + "cache_read_input_token_cost_batches": 2.75e-09, "input_cost_per_token": 5.5e-08, "input_cost_per_token_batches": 2.75e-08, "litellm_provider": "azure", @@ -71585,6 +66341,7 @@ "azure/us/gpt-5.2": { "deprecation_date": "2027-06-08", "cache_read_input_token_cost": 1.925e-07, + "cache_read_input_token_cost_batches": 9.625e-08, "cache_read_input_token_cost_priority": 3.85e-07, "input_cost_per_token": 1.925e-06, "input_cost_per_token_batches": 9.625e-07, @@ -71596,15 +66353,6 @@ "output_cost_per_token_priority": 3.08e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/us/gpt-5.2-chat": { - "deprecation_date": "2026-06-29", - "cache_read_input_token_cost": 1.925e-07, - "input_cost_per_token": 1.925e-06, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 1.54e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/us/gpt-5.2-codex": { "deprecation_date": "2027-07-13", "cache_read_input_token_cost": 1.925e-07, @@ -71623,15 +66371,6 @@ "output_cost_per_token_batches": 9.24e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/us/gpt-5.3-chat": { - "deprecation_date": "2026-06-29", - "cache_read_input_token_cost": 1.925e-07, - "input_cost_per_token": 1.925e-06, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 1.54e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/us/gpt-5.3-codex": { "deprecation_date": "2027-08-24", "cache_read_input_token_cost": 1.925e-07, @@ -71647,6 +66386,7 @@ "azure/us/gpt-5.4-mini": { "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 8.25e-08, + "cache_read_input_token_cost_batches": 4.125e-08, "cache_read_input_token_cost_priority": 1.65e-07, "input_cost_per_token": 8.25e-07, "input_cost_per_token_batches": 4.125e-07, @@ -71661,6 +66401,7 @@ "azure/us/gpt-5.4-nano": { "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2.2e-08, + "cache_read_input_token_cost_batches": 1.1e-08, "input_cost_per_token": 2.2e-07, "input_cost_per_token_batches": 1.1e-07, "litellm_provider": "azure", @@ -71673,11 +66414,13 @@ "deprecation_date": "2027-09-07", "input_cost_per_token": 3.3e-05, "input_cost_per_token_above_272k_tokens": 6.6e-05, + "input_cost_per_token_above_272k_tokens_batches": 3.3e-05, "input_cost_per_token_batches": 1.65e-05, "litellm_provider": "azure", "mode": "chat", "output_cost_per_token": 0.000198, "output_cost_per_token_above_272k_tokens": 0.000297, + "output_cost_per_token_above_272k_tokens_batches": 0.0001485, "output_cost_per_token_batches": 9.9e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, @@ -71691,14 +66434,6 @@ "output_cost_per_token_batches": 2.42e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/us/o1-preview": { - "cache_read_input_token_cost": 8.25e-06, - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/us/o3-deep-research": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 2.75e-06, @@ -72854,6 +67589,16 @@ "supports_reasoning": true, "supports_vision": true }, + "openrouter/typesafe/jev-1.13": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 32000, + "max_output_tokens": 28800, + "max_tokens": 28800, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://openrouter.ai/typesafe/jev-1.13" + }, "typesafe/jev-1.13.0": { "input_cost_per_token": 4.2e-08, "litellm_provider": "typesafe", @@ -72934,16 +67679,16 @@ "supports_web_search": true }, "openrouter/~anthropic/claude-opus-latest": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_1hr": 1e-05, - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 5e-06, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, "litellm_provider": "openrouter", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 2.5e-05, + "output_cost_per_token": 2e-05, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -72978,15 +67723,14 @@ "supports_web_search": true }, "openrouter/~deepseek/deepseek-flash-latest": { - "cache_read_input_token_cost": 6e-09, - "input_cost_per_token": 3e-07, + "cache_read_input_token_cost": 3.6e-09, + "input_cost_per_token": 1.2e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, + "max_output_tokens": 943718, + "max_tokens": 943718, "mode": "chat", - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":1.5e-7,"output_cost_per_token":6e-7,"cache_read_input_token_cost":3e-9}, - "output_cost_per_token": 1.2e-06, + "output_cost_per_token": 4.8e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -72999,15 +67743,14 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-pro-latest": { - "cache_read_input_token_cost": 1.86208e-08, - "input_cost_per_token": 5.58624e-07, + "cache_read_input_token_cost": 1.2726e-08, + "input_cost_per_token": 3.9996e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, + "max_output_tokens": 393216, + "max_tokens": 393216, "mode": "chat", - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.58624e-7,"output_cost_per_token":0.000001675872,"cache_read_input_token_cost":1.86208e-8}, - "output_cost_per_token": 1.675872e-06, + "output_cost_per_token": 1.19988e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73020,14 +67763,14 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-v4-flash-latest": { - "cache_read_input_token_cost": 1.6e-08, - "input_cost_per_token": 4e-08, + "cache_read_input_token_cost": 8e-09, + "input_cost_per_token": 3e-08, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 943718, "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 6.4e-07, + "output_cost_per_token": 8e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73090,13 +67833,13 @@ }, "openrouter/~moonshotai/kimi-latest": { "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "input_cost_per_token": 1.4989e-06, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 943718, "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1.0758e-05, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73134,19 +67877,19 @@ "supports_web_search": true }, "openrouter/~openai/gpt-luna-latest": { - "cache_creation_input_token_cost": 2.5e-07, - "cache_creation_input_token_cost_above_272k_tokens": 5e-07, - "cache_read_input_token_cost": 2e-08, - "cache_read_input_token_cost_above_272k_tokens": 4e-08, - "input_cost_per_token": 2e-07, - "input_cost_per_token_above_272k_tokens": 4e-07, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, "litellm_provider": "openrouter", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_above_272k_tokens": 1.8e-06, + "output_cost_per_token": 5e-07, + "output_cost_per_token_above_272k_tokens": 7.5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73252,14 +67995,14 @@ "supports_web_search": true }, "openrouter/~z-ai/glm-flash-latest": { - "cache_read_input_token_cost": 5e-08, - "input_cost_per_token": 1.5e-07, + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 7.5e-08, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 943718, "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 5e-07, + "output_cost_per_token": 2.5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73272,14 +68015,14 @@ "supports_web_search": false }, "openrouter/~z-ai/glm-latest": { - "cache_read_input_token_cost": 1.69e-07, - "input_cost_per_token": 9.1e-07, + "cache_read_input_token_cost": 1.2142e-07, + "input_cost_per_token": 6.538e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 2.86e-06, + "output_cost_per_token": 2.0548e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73295,7 +68038,7 @@ "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 8e-07, "litellm_provider": "openrouter", - "max_input_tokens": 131072, + "max_input_tokens": 1048576, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -73315,7 +68058,7 @@ "cache_read_input_token_cost": 7.5e-07, "input_cost_per_token": 3e-06, "litellm_provider": "openrouter", - "max_input_tokens": 131072, + "max_input_tokens": 1048576, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -73335,7 +68078,7 @@ "cache_read_input_token_cost": 1.8e-07, "input_cost_per_token": 7e-07, "litellm_provider": "openrouter", - "max_input_tokens": 131072, + "max_input_tokens": 1048576, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -74034,87 +68777,8 @@ "supports_vision": false, "supports_web_search": false }, - "openrouter/deepseek/deepseek-v4-flash-0731:batch": { - "cache_read_input_token_cost": 3.5e-09, - "input_cost_per_token": 1.1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, - "mode": "chat", - "output_cost_per_token": 3.3e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, - "openrouter/deepseek/deepseek-v4-flash-0731:free": { - "input_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 393216, - "max_tokens": 393216, - "mode": "chat", - "output_cost_per_token": 0.0, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": false, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, - "openrouter/deepseek/deepseek-v4-flash-vision-exp:batch": { - "cache_read_input_token_cost": 3.5e-09, - "input_cost_per_token": 1.1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, - "mode": "chat", - "output_cost_per_token": 3.3e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, - "openrouter/deepseek/deepseek-v4-pro-0813:batch": { - "cache_read_input_token_cost": 2.2e-08, - "input_cost_per_token": 6.6e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, - "mode": "chat", - "output_cost_per_token": 1.98e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/dots-studio/dots-3-note-preview:free": { - "deprecation_date": "2026-09-30", + "deprecation_date": "2026-12-31", "input_cost_per_token": 0.0, "litellm_provider": "openrouter", "max_input_tokens": 512000, @@ -74628,26 +69292,6 @@ "supports_vision": false, "supports_web_search": false }, - "openrouter/kwaipilot/kat-coder-pro-v2": { - "cache_read_input_token_cost": 6e-08, - "input_cost_per_token": 3e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 144000, - "max_tokens": 144000, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": false, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/kwaipilot/kat-coder-pro-v2.5": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 7.4e-07, @@ -74727,26 +69371,6 @@ "supports_vision": true, "supports_web_search": false }, - "openrouter/meta/muse-glimmer-30b:batch": { - "cache_read_input_token_cost": 2e-08, - "input_cost_per_token": 1.75e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 131072, - "max_output_tokens": 117964, - "max_tokens": 117964, - "mode": "chat", - "output_cost_per_token": 7.5e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, "openrouter/meta/muse-spark-1.1": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.25e-06, @@ -74885,26 +69509,6 @@ "supports_vision": false, "supports_web_search": false }, - "openrouter/minimax/minimax-m3:batch": { - "cache_read_input_token_cost": 6e-08, - "input_cost_per_token": 3e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 524288, - "max_output_tokens": 471859, - "max_tokens": 471859, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, "openrouter/mistralai/codestral-2508:batch": { "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, @@ -75025,14 +69629,14 @@ "supports_web_search": false }, "openrouter/moonshotai/kimi-k3:batch": { - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "cache_read_input_token_cost": 2.28e-07, + "input_cost_per_token": 2.28e-06, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1.14e-05, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -75083,6 +69687,7 @@ "supports_web_search": false }, "openrouter/nex-agi/nex-n2.5-mini:free": { + "deprecation_date": "2026-09-25", "input_cost_per_token": 0.0, "litellm_provider": "openrouter", "max_input_tokens": 262144, @@ -75102,6 +69707,7 @@ "supports_web_search": false }, "openrouter/nex-agi/nex-n2.5-pro:free": { + "deprecation_date": "2026-09-25", "input_cost_per_token": 0.0, "litellm_provider": "openrouter", "max_input_tokens": 262144, @@ -75852,24 +70458,105 @@ "supports_vision": true, "supports_web_search": true }, - "openrouter/openai/gpt-oss-120b:batch": { - "input_cost_per_token": 1.5e-07, + "openrouter/openai/gpt-6-luna": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, "litellm_provider": "openrouter", - "max_input_tokens": 131072, - "max_output_tokens": 117964, - "max_tokens": 117964, + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 6e-07, + "output_cost_per_token": 5e-07, + "output_cost_per_token_above_272k_tokens": 7.5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": false, + "supports_pdf_input": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false + "supports_vision": true, + "supports_web_search": true + }, + "openrouter/openai/gpt-6-luna-pro": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_above_272k_tokens": 7.5e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "openrouter/openai/gpt-6-sol": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_272k_tokens": 1.5e-05, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "openrouter/openai/gpt-6-sol-pro": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_272k_tokens": 1.5e-05, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true }, "openrouter/openai/o3-mini:batch": { "cache_read_input_token_cost": 2.75e-07, @@ -76045,45 +70732,6 @@ "supports_vision": true, "supports_web_search": true }, - "openrouter/qwen/qwen3.5-9b:batch": { - "input_cost_per_token": 1.7e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 235929, - "max_tokens": 235929, - "mode": "chat", - "output_cost_per_token": 2.5e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": false, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, - "openrouter/qwen/qwen3.8-2.4t-a95b:batch": { - "cache_read_input_token_cost": 2.5e-07, - "input_cost_per_token": 2e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 1010000, - "max_output_tokens": 909000, - "max_tokens": 909000, - "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/qwen/qwen3.8-27b:free": { "input_cost_per_token": 0.0, "litellm_provider": "openrouter", @@ -76616,26 +71264,6 @@ "supports_vision": true, "supports_web_search": false }, - "openrouter/thinkingmachines/inkling:batch": { - "cache_read_input_token_cost": 1.7e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 524288, - "max_output_tokens": 471859, - "max_tokens": 471859, - "mode": "chat", - "output_cost_per_token": 4.05e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, "openrouter/thinkingmachines/inkling:free": { "input_cost_per_token": 0.0, "litellm_provider": "openrouter", @@ -76757,35 +71385,15 @@ "supports_vision": true, "supports_web_search": true }, - "openrouter/z-ai/glm-5.2:batch": { - "cache_read_input_token_cost": 7e-08, - "input_cost_per_token": 7e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, - "mode": "chat", - "output_cost_per_token": 2.2e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/z-ai/glm-5.3-flash:batch": { - "cache_read_input_token_cost": 1.5e-08, - "input_cost_per_token": 7.5e-08, + "cache_read_input_token_cost": 1.2e-08, + "input_cost_per_token": 6e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 2.5e-07, + "output_cost_per_token": 2e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -76798,14 +71406,14 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3:batch": { - "cache_read_input_token_cost": 1.3e-07, - "input_cost_per_token": 7e-07, + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 7.2e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 2.2e-06, + "output_cost_per_token": 2.4e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -77044,5 +71652,751 @@ "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true + }, + "xai/grok-4.20-0309": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-0309": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-latest-non-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-latest-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-non-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-0304": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-0304-non-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-0304-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-non-reasoning-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-reasoning-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-multi-agent-beta-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "responses", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true, + "supported_endpoints": [ + "/v1/responses" + ] + }, + "xai/grok-4.20-multi-agent-experimental-beta-0304": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "responses", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true, + "supported_endpoints": [ + "/v1/responses" + ] + }, + "xai/grok-4.20-multi-agent-experimental-beta-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "responses", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true, + "supported_endpoints": [ + "/v1/responses" + ] + }, + "xai/grok-4.20-non-reasoning-gv2": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-reasoning-gv2": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "openrouter/nex-agi/nex-n2.5-mini": { + "cache_read_input_token_cost": 2.5e-09, + "input_cost_per_token": 2.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "output_cost_per_token": 1e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": false, + "supports_pdf_input": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": false + }, + "openrouter/nex-agi/nex-n2.5-pro": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 7.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": false + }, + "baseten/deepseek-ai/DeepSeek-V4.1-Flash": { + "cache_read_input_token_cost": 7e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/moonshotai/Kimi-K2.6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "baseten", + "max_input_tokens": 262000, + "max_output_tokens": 262000, + "max_tokens": 262000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/moonshotai/Kimi-K2.7-Code": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "baseten", + "max_input_tokens": 262000, + "max_output_tokens": 262000, + "max_tokens": 262000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/moonshotai/Kimi-K3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "baseten", + "max_input_tokens": 202800, + "max_output_tokens": 202800, + "max_tokens": 202800, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "baseten/thinkingmachines/inkling": { + "cache_read_input_token_cost": 1.7e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4.05e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/thinkingmachines/inkling-small": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 5e-07, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/zai-org/GLM-5.2": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/zai-org/GLM-5.3-Flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/deepseek-ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.3e-07, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.6e-07, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "baseten/deepseek-ai/DeepSeek-V4-Pro": { + "cache_read_input_token_cost": 1.45e-07, + "input_cost_per_token": 1.74e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.48e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "baseten/deepseek-ai/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 1.32e-07, + "input_cost_per_token": 1.32e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "baseten/zai-org/GLM-5.2-Fast": { + "cache_read_input_token_cost": 2.1e-07, + "input_cost_per_token": 2.1e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "xai/grok-code-fast": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "xai/grok-code-fast-1": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "xai/grok-code-fast-1-0825": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } diff --git a/litellm/models/mcp_server.py b/litellm/models/mcp_server.py index 7ccff9434a7..9125d708e79 100644 --- a/litellm/models/mcp_server.py +++ b/litellm/models/mcp_server.py @@ -50,6 +50,7 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase): """Represents a LiteLLM_MCPServerTable record""" server_id: str + is_config: bool = Field(default=False, description="Whether this server is defined in config and is read-only.") server_name: str | None = None alias: str | None = None description: str | None = None diff --git a/litellm/ocr/dispatch.py b/litellm/ocr/dispatch.py index 55b19458b7a..b26175c943b 100644 --- a/litellm/ocr/dispatch.py +++ b/litellm/ocr/dispatch.py @@ -6,7 +6,7 @@ import httpx from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.ocr import main from litellm.ocr.main import convert_file_document_to_url_document, get_mime_type -from litellm.rust_bridge.catalog import Context, Route +from litellm.rust_bridge.catalog import Route, RouteContext from litellm.rust_bridge.dispatch import PublicDispatch, call_hook from litellm.rust_bridge.ocr.entrypoints import NATIVE_AOCR, NATIVE_OCR, LiteLLMOcrRequest @@ -52,10 +52,10 @@ _PYTHON_AOCR: Final = cast( # cast-ok: forward the original call shape through ) -def _context(request: LiteLLMOcrRequest) -> Context: +def _context(request: LiteLLMOcrRequest) -> RouteContext: prefix, separator, _ = request.model.partition("/") provider: Final = request.custom_llm_provider or (prefix if separator else None) - return Context(Route.OCR, provider=provider, model=request.model) + return RouteContext(Route.OCR, provider=provider, model=request.model) _DISPATCH: Final = PublicDispatch( diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 2e5cad29b0a..d6478421834 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -663,6 +663,8 @@ class MCPRequestHandler: # with ``server.py::_get_mcp_servers_in_path``, which also accepts the # un-rewritten form (some entry points may skip the # ``dynamic_mcp_route`` rewrite). + if path.rstrip("/") in ("/mcp/sse", "/mcp/sse/messages"): + return [] segments: Final = [s for s in path.split("/") if s] if len(segments) >= 2 and segments[1] == "mcp" and segments[0] != "mcp": return [segments[0]] @@ -966,10 +968,11 @@ class MCPRequestHandler: on top of these direct grants, each source bounded by ITS OWN org, so a user spanning organizations cannot leak one org's servers past another's ceiling. - Error handling: ``get_user_object`` catches every DB failure and re-raises a bare ``ValueError``, so a - missing user and a real outage look identical (the cause survives only as ``__context__``). - ``_raise_503_if_db_unavailable`` walks the cause chain so an outage stays a retryable 503 while any - other failure fails closed as 401, not an opaque 500; the object-permission load shares that boundary.""" + Error handling: ``get_user_object`` lets a database outage propagate as-is and re-raises every other + DB failure as a bare ``ValueError`` (the cause surviving only as ``__context__``). + ``_raise_503_if_db_unavailable`` walks the cause chain so an outage stays a retryable 503 whichever + shape it arrives in, while any other failure fails closed as 401, not an opaque 500; the + object-permission load shares that boundary.""" from litellm.proxy.auth.auth_checks import get_object_permission, get_user_object from litellm.proxy.proxy_server import prisma_client, user_api_key_cache @@ -1116,9 +1119,8 @@ class MCPRequestHandler: (401) or surface as an opaque 500; the caller retries. Mirrors ``UserAPIKeyAuthExceptionHandler``, which renders a service-unavailable database error as 503 on the standard pipeline. - Classifies across the ``__cause__``/``__context__`` chain, not just ``e`` itself: ``get_user_object`` - re-raises every DB failure as a bare ``ValueError``, so a type-based check on the top exception - would miss a real outage wrapped inside it.""" + Classifies across the ``__cause__``/``__context__`` chain, not just ``e`` itself, so an outage a + caller re-raised inside a domain exception is still recognized.""" from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler outage: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(e) diff --git a/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py b/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py index 37a893973e3..6ae33cc1629 100644 --- a/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py +++ b/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py @@ -276,9 +276,9 @@ async def load_active_user_by_id( database-service-unavailable error is a retryable outage (``unavailable``). Everything else fails closed as ``no_active_key`` (the caller maps it to invalid_grant): a ``ProxyException`` / ``HTTPException``, a SCIM-deactivated user, and, unlike the key path, a missing user. ``get_user_object`` - catches every DB failure and re-raises a bare ``ValueError`` (a deleted user and a real outage look - identical, the original error surviving only as ``__context__``), so the outage check walks the cause - chain, and a missing user falls through to ``no_active_key`` rather than an opaque gateway fault. + lets a real outage propagate as-is and re-raises any other DB failure as a bare ``ValueError`` (the + original error surviving only as ``__context__``), so the outage check walks the cause chain, and a + missing user falls through to ``no_active_key`` rather than an opaque gateway fault. ``source="database"`` reads the row from the database, never the cache, so the credential mint refuses a user that a writer deactivated or deleted without evicting the cached row, and it leaves the fresh row in the cache for the requests the credential makes next. Every other caller keeps the cache read, diff --git a/litellm/proxy/_experimental/mcp_server/idp_token_exchange.py b/litellm/proxy/_experimental/mcp_server/idp_token_exchange.py index 80868296b50..a437df17e6a 100644 --- a/litellm/proxy/_experimental/mcp_server/idp_token_exchange.py +++ b/litellm/proxy/_experimental/mcp_server/idp_token_exchange.py @@ -196,9 +196,8 @@ def _check_unavailable_description(outage: GatewayOutage) -> str: def _gateway_could_not_verify(denied: Exception) -> GatewayOutage | None: - """A database fault anywhere in the chain (``get_user_object`` wraps prisma failures in a - bare ``ValueError``) or a 5xx from JWT auth (the IdP's JWKS unreachable with no cached - copy) is the gateway failing, not the token. A fault retrying cannot clear (a missing or + """A database fault anywhere in the chain or a 5xx from JWT auth (the IdP's JWKS + unreachable with no cached copy) is the gateway failing, not the token. A fault retrying cannot clear (a missing or version-skewed query engine) is named as such, the way the mint path words it, so the client is not told to wait on a deployment that needs repair.""" fault: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(denied) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 4a2713cb19c..20c114a2f3e 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -7081,6 +7081,7 @@ class MCPServerManager: def _build_mcp_server_table(self, server: MCPServer) -> LiteLLM_MCPServerTable: return LiteLLM_MCPServerTable( server_id=server.server_id, + is_config=self.is_config_declared_server(server.server_id) and server.server_id not in self.registry, server_name=server.server_name, alias=server.alias, description=(server.mcp_info.get("description") if server.mcp_info else None), diff --git a/litellm/proxy/_experimental/mcp_server/operations.py b/litellm/proxy/_experimental/mcp_server/operations.py index fcee3483e15..2bd6d186f24 100644 --- a/litellm/proxy/_experimental/mcp_server/operations.py +++ b/litellm/proxy/_experimental/mcp_server/operations.py @@ -2198,7 +2198,7 @@ async def _fire_mcp_tool_call_logging( from litellm.proxy.proxy_server import proxy_logging_obj logging_obj.post_call(original_response=result) - await logging_obj.async_post_mcp_tool_call_hook( + result = await logging_obj.async_post_mcp_tool_call_hook( kwargs=logging_obj.model_call_details, response_obj=result, start_time=start_time, diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 433b693fcae..fe508fc22c7 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -21,6 +21,7 @@ from fastapi import FastAPI, HTTPException from pydantic import ConfigDict, TypeAdapter, ValidationError from starlette.requests import Request as StarletteRequest from starlette.responses import JSONResponse +from starlette.routing import Route from starlette.types import Message, Receive, Scope, Send from litellm._logging import verbose_logger @@ -474,6 +475,8 @@ if MCP_AVAILABLE: AuthContextMiddleware, auth_context_var, ) + from mcp.server.auth.middleware.bearer_auth import AuthenticatedUser + from mcp.server.auth.provider import AccessToken from mcp.server.context import ServerRequestContext from mcp.server.lowlevel.server import NotificationOptions from mcp.server.models import InitializationOptions @@ -573,7 +576,7 @@ if MCP_AVAILABLE: version=LITELLM_MCP_SERVER_VERSION, ) server.create_initialization_options = types.MethodType(_gateway_create_initialization_options, server) - sse: Final[SseServerTransport] = SseServerTransport("/mcp/sse/messages") + sse: Final[SseServerTransport] = SseServerTransport("/sse/messages") # Create session managers session_manager_stateless: Final = StreamableHTTPSessionManager( @@ -629,18 +632,9 @@ if MCP_AVAILABLE: # Keep this alias so existing references to session_manager still work session_manager: Final = session_manager_stateless - # Create SSE session manager - sse_session_manager: Final = StreamableHTTPSessionManager( - app=server, - event_store=None, - json_response=False, # Use SSE responses for this endpoint - stateless=True, - ) - # Context managers for proper lifecycle management _session_manager_cm = None _session_manager_stateful_cm = None - _sse_session_manager_cm = None _stateful_auth_context_cleanup_task: asyncio.Task | None = None async def _purge_expired_stateful_session_auth_contexts( @@ -732,7 +726,6 @@ if MCP_AVAILABLE: _SESSION_MANAGERS_INITIALIZED, \ _session_manager_cm, \ _session_manager_stateful_cm, \ - _sse_session_manager_cm, \ _stateful_auth_context_cleanup_task # Use async lock to prevent concurrent initialization @@ -745,12 +738,10 @@ if MCP_AVAILABLE: # Start the session managers with context managers _session_manager_cm = session_manager_stateless.run() _session_manager_stateful_cm = session_manager_stateful.run() - _sse_session_manager_cm = sse_session_manager.run() # Enter the context managers await _session_manager_cm.__aenter__() await _session_manager_stateful_cm.__aenter__() - await _sse_session_manager_cm.__aenter__() _stateful_auth_context_cleanup_task = asyncio.create_task(_cleanup_expired_stateful_session_auth_contexts()) _SESSION_MANAGERS_INITIALIZED = True @@ -762,7 +753,6 @@ if MCP_AVAILABLE: _SESSION_MANAGERS_INITIALIZED, \ _session_manager_cm, \ _session_manager_stateful_cm, \ - _sse_session_manager_cm, \ _stateful_auth_context_cleanup_task if _SESSION_MANAGERS_INITIALIZED: @@ -773,8 +763,6 @@ if MCP_AVAILABLE: _stateful_auth_context_cleanup_task.cancel() with contextlib.suppress(asyncio.CancelledError): await _stateful_auth_context_cleanup_task - if _sse_session_manager_cm: - await _sse_session_manager_cm.__aexit__(None, None, None) if _session_manager_stateful_cm: await _session_manager_stateful_cm.__aexit__(None, None, None) if _session_manager_cm: @@ -784,7 +772,6 @@ if MCP_AVAILABLE: _session_manager_cm = None _session_manager_stateful_cm = None - _sse_session_manager_cm = None _stateful_auth_context_cleanup_task = None _SESSION_MANAGERS_INITIALIZED = False @@ -1054,6 +1041,8 @@ if MCP_AVAILABLE: """ import re + if path.rstrip("/") in ("/mcp/sse", "/mcp/sse/messages"): + return None mcp_servers_from_path: list[str] | None = None segments: Final = [s for s in path.split("/") if s] if len(segments) >= 2 and segments[1] == "mcp" and segments[0] != "mcp": @@ -2286,7 +2275,24 @@ if MCP_AVAILABLE: async def handle_sse_mcp(scope: Scope, receive: Receive, send: Send) -> None: """Handle MCP requests through SSE.""" try: - path: Final[str] = scope.get("path", "") + bad_version: Final = unsupported_protocol_version(scope) + if bad_version is not None: + supported: Final = ", ".join(sorted(HANDSHAKE_PROTOCOL_VERSIONS)) + await JSONResponse( + status_code=400, + content={ # mutable-ok: JSON-RPC error payload + "jsonrpc": "2.0", + "id": None, + "error": { + "code": INVALID_REQUEST, + "message": f"Unsupported MCP-Protocol-Version {bad_version}; supported: {supported}", + }, + }, + )(scope, receive, send) + return + from litellm.proxy.auth.auth_utils import get_request_route + + path: Final = get_request_route(StarletteRequest(scope)) ( user_api_key_auth, mcp_auth_header, @@ -2353,9 +2359,14 @@ if MCP_AVAILABLE: client_ip=_sse_client_ip, ) - if not _SESSION_MANAGERS_INITIALIZED: - await initialize_session_managers() - await asyncio.sleep(0.1) + owner: Final = _owner_fingerprint_for(user_api_key_auth, oauth2_headers, _sse_client_ip) + transport_scope: Final[Scope] = { + **scope, + "user": AuthenticatedUser(AccessToken(token=owner, client_id=owner, scopes=[])), + } + if scope["method"] == "POST": + await sse.handle_post_message(transport_scope, receive, send) + return async with _gateway_initialize_instructions_request_scope( user_api_key_auth, @@ -2364,7 +2375,8 @@ if MCP_AVAILABLE: scoped_server_endpoint=scoped_server_endpoint, is_initialize=scope.get("method") == "GET", ): - await sse_session_manager.handle_request(scope, receive, send) + async with sse.connect_sse(transport_scope, receive, send) as (read_stream, write_stream): + await server.run(read_stream, write_stream, server.create_initialization_options()) except MCPUpstreamAuthError as e: # Upstream delegated auth returned 401; surface it to the client so # standards-compliant MCP clients trigger the upstream OAuth flow. @@ -2387,7 +2399,6 @@ if MCP_AVAILABLE: # Try to send a graceful error response for non-HTTP exceptions try: # Send a proper HTTP error response instead of letting the exception bubble up - from starlette.responses import JSONResponse from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR error_response: Final = JSONResponse( @@ -2418,11 +2429,22 @@ if MCP_AVAILABLE: """ return {"enabled": MCP_AVAILABLE} + class _LegacySseEndpoint: + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + await handle_sse_mcp(scope, receive, send) + + for sse_path, sse_method in ( + ("/sse", "GET"), + ("/sse/", "GET"), + ("/sse/messages", "POST"), + ("/sse/messages/", "POST"), + ): + app.router.routes.append(Route(sse_path, endpoint=_LegacySseEndpoint(), methods=[sse_method])) + # Mount the MCP handlers app.mount("/", handle_streamable_http_mcp) app.mount("/mcp", handle_streamable_http_mcp) app.mount("/{mcp_server_name}/mcp", handle_streamable_http_mcp) - app.mount("/sse", handle_sse_mcp) app.add_middleware(AuthContextMiddleware) ######################################################## diff --git a/litellm/proxy/_experimental/mcp_server/sse_transport.py b/litellm/proxy/_experimental/mcp_server/sse_transport.py index 2a08a5f8f7a..839d2eebbbe 100644 --- a/litellm/proxy/_experimental/mcp_server/sse_transport.py +++ b/litellm/proxy/_experimental/mcp_server/sse_transport.py @@ -1,138 +1,3 @@ -""" -This is a modification of code from: https://github.com/SecretiveShell/MCP-Bridge/blob/master/mcp_bridge/mcp_server/sse_transport.py +from mcp.server.sse import SseServerTransport -Credit to the maintainers of SecretiveShell for their SSE Transport implementation - -""" - -from contextlib import asynccontextmanager -from typing import Any, Final -from urllib.parse import quote -from uuid import UUID, uuid4 - -import anyio -from anyio.streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStream -from fastapi.requests import Request -from fastapi.responses import Response -from mcp import types -from pydantic import ValidationError -from sse_starlette import EventSourceResponse -from starlette.types import Receive, Scope, Send - -from litellm._logging import verbose_logger - - -class SseServerTransport: - """ - SSE server transport for MCP. This class provides _two_ ASGI applications, - suitable to be used with a framework like Starlette and a server like Hypercorn: - - 1. connect_sse() is an ASGI application which receives incoming GET requests, - and sets up a new SSE stream to send server messages to the client. - 2. handle_post_message() is an ASGI application which receives incoming POST - requests, which should contain client messages that link to a - previously-established SSE session. - """ - - _endpoint: str - _read_stream_writers: dict[UUID, MemoryObjectSendStream[types.JSONRPCMessage | Exception]] - - def __init__(self, endpoint: str) -> None: - """ - Creates a new SSE server transport, which will direct the client to POST - messages to the relative or absolute URL given. - """ - - super().__init__() - self._endpoint = endpoint - self._read_stream_writers = {} - verbose_logger.debug("SseServerTransport initialized with endpoint: %s", endpoint) - - @asynccontextmanager - async def connect_sse(self, request: Request): - if request.scope["type"] != "http": - verbose_logger.error("connect_sse received non-HTTP request") - raise ValueError("connect_sse can only handle HTTP requests") - - verbose_logger.debug("Setting up SSE connection") - read_stream: MemoryObjectReceiveStream[types.JSONRPCMessage | Exception] - read_stream_writer: MemoryObjectSendStream[types.JSONRPCMessage | Exception] - - write_stream: MemoryObjectSendStream[types.JSONRPCMessage] - write_stream_reader: MemoryObjectReceiveStream[types.JSONRPCMessage] - - read_stream_writer, read_stream = anyio.create_memory_object_stream(0) - write_stream, write_stream_reader = anyio.create_memory_object_stream(0) - - session_id: Final = uuid4() - session_uri: Final = f"{quote(self._endpoint)}?session_id={session_id.hex}" - self._read_stream_writers[session_id] = read_stream_writer - verbose_logger.debug("Created new session with ID: %s", session_id) - - sse_stream_writer: MemoryObjectSendStream[dict[str, Any]] - sse_stream_reader: MemoryObjectReceiveStream[dict[str, Any]] - sse_stream_writer, sse_stream_reader = anyio.create_memory_object_stream(0, dict[str, Any]) - - async def sse_writer(): - verbose_logger.debug("Starting SSE writer") - async with sse_stream_writer, write_stream_reader: - await sse_stream_writer.send({"event": "endpoint", "data": session_uri}) - verbose_logger.debug("Sent endpoint event: %s", session_uri) - - async for message in write_stream_reader: - verbose_logger.debug("Sending message via SSE: %s", message) - await sse_stream_writer.send( - { - "event": "message", - "data": message.model_dump_json(by_alias=True, exclude_none=True), - } - ) - - async with anyio.create_task_group() as tg: - response: Final = EventSourceResponse(content=sse_stream_reader, data_sender_callable=sse_writer) - verbose_logger.debug("Starting SSE response task") - tg.start_soon(response, request.scope, request.receive, request._send) - - verbose_logger.debug("Yielding read and write streams") - yield (read_stream, write_stream) - - async def handle_post_message(self, scope: Scope, receive: Receive, send: Send) -> Response: - verbose_logger.debug("Handling POST message") - request: Final = Request(scope, receive) - - session_id_param: Final = request.query_params.get("session_id") - if session_id_param is None: - verbose_logger.warning("Received request without session_id") - response = Response("session_id is required", status_code=400) - return response - - try: - session_id: Final = UUID(hex=session_id_param) - verbose_logger.debug("Parsed session ID: %s", session_id) - except ValueError: - verbose_logger.warning("Received invalid session ID: %s", session_id_param) - response = Response("Invalid session ID", status_code=400) - return response - - writer: Final = self._read_stream_writers.get(session_id) - if not writer: - verbose_logger.warning("Could not find session for ID: %s", session_id) - response = Response("Could not find session", status_code=404) - return response - - json: Final = await request.json() - verbose_logger.debug("Received JSON: %s", json) - - try: - message: Final = types.JSONRPCMessage.model_validate(json) - verbose_logger.debug("Validated client message: %s", message) - except ValidationError as err: - verbose_logger.error("Failed to parse message: %s", err) - response = Response("Could not parse message", status_code=400) - await writer.send(err) - return response - - verbose_logger.debug("Sending message to writer: %s", message) - response = Response("Accepted", status_code=202) - await writer.send(message) - return response +__all__ = ("SseServerTransport",) diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/_experimental/mcp_server/utils.py index 6bd080f5216..7411dc5c4f0 100644 --- a/litellm/proxy/_experimental/mcp_server/utils.py +++ b/litellm/proxy/_experimental/mcp_server/utils.py @@ -986,7 +986,7 @@ def _forwarded_upstream_header_names() -> frozenset[str]: ) -def _upstream_credential_headers(header_names: Iterable[str]) -> frozenset[str]: +def upstream_credential_headers(header_names: Iterable[str]) -> frozenset[str]: """Lowercased names of the headers in ``header_names`` that carry an upstream MCP credential rather than request context: the configured client side auth header, any header name a configured server forwards upstream via ``extra_headers``, and the @@ -1038,7 +1038,7 @@ def build_synthetic_mcp_request( custom_key_header: Final = _custom_litellm_key_header_name() excluded: Final = ( _SYNTHETIC_REQUEST_EXCLUDED_HEADERS - | _upstream_credential_headers(raw_headers.keys() if raw_headers else ()) + | upstream_credential_headers(raw_headers.keys() if raw_headers else ()) | (frozenset({custom_key_header.lower()}) if custom_key_header else frozenset()) ) forwarded: Final = tuple( @@ -1086,7 +1086,7 @@ def logging_safe_mcp_headers(raw_headers: Mapping[str, str] | None) -> Mapping[s ) excluded: Final = ( - _upstream_credential_headers(raw_headers.keys() if raw_headers else ()) + upstream_credential_headers(raw_headers.keys() if raw_headers else ()) | UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS | frozenset({"host"}) ) diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 17c85bbdbca..5be87a8bf4d 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -203,6 +203,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( "/cursor/", "/deepgram/", "/eu.assemblyai/", + "/fal_ai/", "/gemini/", "/gigachat/", "/milvus/", @@ -210,8 +211,10 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( "/nvidia_nim/", "/openai/", "/openai_passthrough/", + "/tinyfish/", "/transcribe", "/typesafe/", + "/openrouter/", "/vertex-ai/", "/vertex_ai/", "/vllm/", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 1293a05e4a6..87f28624d58 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -15424,14 +15424,31 @@ "title": "Jwt Issuer" }, "key": { - "title": "Key", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Key" + }, + "token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Token" } }, "required": [ "jwt_claim_name", - "jwt_claim_value", - "key" + "jwt_claim_value" ], "title": "CreateJWTKeyMappingRequest", "type": "object" @@ -15595,6 +15612,17 @@ } ], "title": "Key" + }, + "token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Token" } }, "required": [ @@ -18490,6 +18518,223 @@ ] } }, + "/fal_ai/{endpoint}": { + "delete": { + "operationId": "fal_ai_proxy_route_fal_ai__endpoint__delete", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fal Ai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "get": { + "operationId": "fal_ai_proxy_route_fal_ai__endpoint__get", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fal Ai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "patch": { + "operationId": "fal_ai_proxy_route_fal_ai__endpoint__patch", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fal Ai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "post": { + "operationId": "fal_ai_proxy_route_fal_ai__endpoint__post", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fal Ai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "put": { + "operationId": "fal_ai_proxy_route_fal_ai__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Fal Ai Proxy Route", + "tags": [ + "llm_passthrough" + ] + } + }, "/gemini/{endpoint}": { "delete": { "description": "[Docs](https://docs.litellm.ai/docs/pass_through/google_ai_studio)", @@ -19674,7 +19919,7 @@ } } }, - "description": "\n Unified rate-limit error.\n\n Every rate-limit condition surfaced by litellm \u2014 whether it originated from\n an upstream LLM provider, a vendor batch endpoint, or one of litellm's own\n proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\n max-iterations, etc.) \u2014 is raised as an instance of this class.\n\n The :attr:`category` attribute lets callers distinguish the source. See\n :class:`RateLimitErrorCategory` for the available values.\n " + "description": "Unified rate-limit error.\n\nEvery rate-limit condition surfaced by litellm \u2014 whether it originated from\nan upstream LLM provider, a vendor batch endpoint, or one of litellm's own\nproxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\nmax-iterations, etc.) \u2014 is raised as an instance of this class.\n\nThe :attr:`category` attribute lets callers distinguish the source. See\n:class:`RateLimitErrorCategory` for the available values." }, "500": { "content": { @@ -20701,6 +20946,313 @@ ] } }, + "/openrouter/{endpoint}": { + "delete": { + "operationId": "openrouter_proxy_route_openrouter__endpoint__delete", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Openrouter Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "get": { + "operationId": "openrouter_proxy_route_openrouter__endpoint__get", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Openrouter Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "patch": { + "operationId": "openrouter_proxy_route_openrouter__endpoint__patch", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Openrouter Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "post": { + "operationId": "openrouter_proxy_route_openrouter__endpoint__post", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Openrouter Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "put": { + "operationId": "openrouter_proxy_route_openrouter__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Openrouter Proxy Route", + "tags": [ + "llm_passthrough" + ] + } + }, + "/tinyfish/{endpoint}": { + "get": { + "description": "Pass-through for the TinyFish Agent API (goal-based web automation).\n\nForwarded endpoints:\n- POST /v1/automation/run \u2014 run to completion (blocking)\n- POST /v1/automation/run-async \u2014 submit a run, poll GET /v1/runs/{id} for the result\n- POST /v1/automation/run-sse \u2014 run with SSE progress events\n- GET /v1/runs/{id} \u2014 run status / result\n- POST /v1/runs/{id}/cancel \u2014 cancel a run\n\nEvery other Agent API endpoint (vault, wallet, browser profiles, and the GET /v1/runs\nlisting, which would let any caller discover other callers' run ids) returns 403: all\nproxy callers share one upstream key.\n\nCredential lookup order:\n1. passthrough_endpoint_router (config.yaml deployments with use_in_pass_through)\n2. TINYFISH_API_KEY environment variable\n\n[Docs](https://docs.litellm.ai/docs/pass_through/tinyfish)", + "operationId": "tinyfish_proxy_route_tinyfish__endpoint__get", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Tinyfish Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "post": { + "description": "Pass-through for the TinyFish Agent API (goal-based web automation).\n\nForwarded endpoints:\n- POST /v1/automation/run \u2014 run to completion (blocking)\n- POST /v1/automation/run-async \u2014 submit a run, poll GET /v1/runs/{id} for the result\n- POST /v1/automation/run-sse \u2014 run with SSE progress events\n- GET /v1/runs/{id} \u2014 run status / result\n- POST /v1/runs/{id}/cancel \u2014 cancel a run\n\nEvery other Agent API endpoint (vault, wallet, browser profiles, and the GET /v1/runs\nlisting, which would let any caller discover other callers' run ids) returns 403: all\nproxy callers share one upstream key.\n\nCredential lookup order:\n1. passthrough_endpoint_router (config.yaml deployments with use_in_pass_through)\n2. TINYFISH_API_KEY environment variable\n\n[Docs](https://docs.litellm.ai/docs/pass_through/tinyfish)", + "operationId": "tinyfish_proxy_route_tinyfish__endpoint__post", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Tinyfish Proxy Route", + "tags": [ + "llm_passthrough" + ] + } + }, "/transcribe": { "post": { "description": "AWS-SDK-shaped pass-through for Amazon Transcribe: point the SDK's `endpoint_url`\nat `/transcribe` and the operation is read from the `X-Amz-Target` header, per the\nAWS JSON 1.1 protocol.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/transcribe)", @@ -24215,6 +24767,12 @@ "title": "Is Byok", "type": "boolean" }, + "is_config": { + "default": false, + "description": "Whether this server is defined in config and is read-only.", + "title": "Is Config", + "type": "boolean" + }, "issuer": { "anyOf": [ { @@ -27265,6 +27823,12 @@ "title": "Is Byok", "type": "boolean" }, + "is_config": { + "default": false, + "description": "Whether this server is defined in config and is read-only.", + "title": "Is Config", + "type": "boolean" + }, "issuer": { "anyOf": [ { diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 3f2d2f57fd4..76df8790b92 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -491,14 +491,17 @@ class LiteLLMRoutes(enum.Enum): "/openai_passthrough", "/assemblyai", "/eu.assemblyai", + "/tinyfish", "/vllm", "/mistral", "/typesafe", + "/openrouter", "/milvus", "/gigachat", "/watsonx", "/nvidia_nim", "/deepgram", + "/fal_ai", ] ######################################################### @@ -530,6 +533,9 @@ class LiteLLMRoutes(enum.Enum): mcp_inference_routes = [ "/mcp", "/mcp/", + "/mcp/sse/", + "/mcp/sse/messages", + "/mcp/sse/messages/", "/mcp/proxy", "/mcp/{subpath}", "/mcp/tools", @@ -2648,6 +2654,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="opt-in to RFC 8628 verification_uri_complete for the CLI SSO device flow, pre-filling the user_code in the browser. Off by default; intended for same-host clients where the device that starts the flow and the browser run on the same machine", ) + include_call_id_in_error_body: bool | None = Field( + None, + description="opt-in to copy the x-litellm-call-id response header's value into JSON error bodies, as error.litellm_call_id on the OpenAI-shaped and /v1/messages routes and as a top-level litellm_call_id on pass-through routes, so an error a client prints names the request to look up. Off by default", + ) enable_claude_code_gateway: bool | None = Field( None, description="serve the Claude Code gateway protocol (https://code.claude.com/docs/en/claude-apps-gateway) under /claude_code_gateway: OAuth device-flow sign-in reusing proxy SSO, plus managed settings and OTLP telemetry ingestion. Off by default", @@ -4733,7 +4743,8 @@ class KeyHealthResponse(TypedDict, total=False): class CreateJWTKeyMappingRequest(LiteLLMPydanticObjectBase): jwt_claim_name: str jwt_claim_value: str - key: str + key: str | None = None + token: str | None = None jwt_issuer: str | None = None description: str | None = None @@ -4741,6 +4752,7 @@ class CreateJWTKeyMappingRequest(LiteLLMPydanticObjectBase): class UpdateJWTKeyMappingRequest(LiteLLMPydanticObjectBase): id: str key: str | None = None + token: str | None = None jwt_issuer: str | None = None description: str | None = None is_active: bool | None = None diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 644778bcb9f..d9558b86e95 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -8,7 +8,11 @@ from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import JSONResponse import litellm -from litellm.anthropic_interface.exceptions import AnthropicErrorResponse, AnthropicExceptionMapping +from litellm.anthropic_interface.exceptions import ( + AnthropicErrorDetail, + AnthropicErrorResponse, + AnthropicExceptionMapping, +) from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.llms.anthropic.experimental_pass_through.context_management import ( AnthropicContextManagementError, @@ -25,8 +29,10 @@ from litellm.proxy.common_request_processing import ( proxy_exception_from_http_exception, resolve_litellm_call_id, ) +from litellm.proxy.common_utils.error_body_call_id import error_body_call_id from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.common_utils.openai_error_payload import ( + LITELLM_CALL_ID_HEADER, error_status_code, openai_error_param, openai_error_type, @@ -37,9 +43,29 @@ from litellm.types.utils import TokenCountResponse router: Final = APIRouter() +def _with_provider_specific_fields(exc: ProxyException, detail: AnthropicErrorDetail) -> AnthropicErrorDetail: + if not exc.provider_specific_fields: + return detail + with_fields: Final[AnthropicErrorDetail] = {**detail, "provider_specific_fields": exc.provider_specific_fields} + return with_fields + + +def _anthropic_error_detail( + exc: ProxyException, detail: AnthropicErrorDetail, call_id: str | None +) -> AnthropicErrorDetail: + if call_id is None: + return _with_provider_specific_fields(exc, detail) + with_call_id: Final[AnthropicErrorDetail] = { + **_with_provider_specific_fields(exc, detail), + "litellm_call_id": call_id, + } + return with_call_id + + def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse: from litellm.proxy.proxy_server import ( _close_dangling_otel_server_span, # pyright: ignore[reportPrivateUsage] # proxy_server keeps the span-close helper private; error JSONResponses returned by the route must stamp the OTel server span like the global ProxyException handler does + general_settings_view, ) status_code: Final = int(exc.code) if exc.code is not None and exc.code.isdigit() else 500 @@ -49,11 +75,10 @@ def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSO raw_message=exc.message, request_id=request.headers.get("x-request-id"), ) - if not exc.provider_specific_fields: - return JSONResponse(status_code=status_code, content=envelope, headers=exc.headers) + body_call_id: Final = error_body_call_id(general_settings_view(), exc.headers.get(LITELLM_CALL_ID_HEADER)) content: Final[AnthropicErrorResponse] = { **envelope, - "error": {**envelope["error"], "provider_specific_fields": exc.provider_specific_fields}, + "error": _anthropic_error_detail(exc, envelope["error"], body_call_id), } return JSONResponse(status_code=status_code, content=content, headers=exc.headers) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index b3efde7f05f..d2fed1eb421 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -2697,9 +2697,15 @@ async def get_user_object( raise except Exception as e: _log_budget_lookup_failure("user", e) - raise ValueError( - f"User doesn't exist in db. 'user_id'={user_id}. Create user via `/user/new` call. Got error - {e}" - ) + raise _user_read_failure(user_id=user_id, error=e) + + +def _user_read_failure(user_id: str, error: Exception) -> Exception: + if PrismaDBExceptionHandler.is_database_service_unavailable_error(error): + return error + return ValueError( + f"User doesn't exist in db. 'user_id'={user_id}. Create user via `/user/new` call. Got error - {error}" + ) async def _cache_management_object( @@ -4304,7 +4310,10 @@ def _can_object_call_model( ) return True - potential_models: Final = [model] + from litellm.router_strategy.complexity_router.context_compaction import native_compaction_parent + + compaction_parent: Final = native_compaction_parent(model) + potential_models: Final = [model, compaction_parent] if compaction_parent is not None else [model] if model in litellm.model_alias_map: potential_models.append(litellm.model_alias_map[model]) elif llm_router and model in llm_router.model_group_alias: diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index bbe4b0f5c35..9bf7f6cab96 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -71,12 +71,7 @@ def _as_proxy_exception(e: Exception) -> ProxyException: if isinstance(e, ProxyException): return e if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): - return ProxyException( - message=PrismaDBExceptionHandler.database_unavailable_message(e), - type=ProxyErrorTypes.no_db_connection, - param="None", - code=status.HTTP_503_SERVICE_UNAVAILABLE, - ) + return PrismaDBExceptionHandler.service_unavailable_proxy_exception(e) return ProxyException( message="Authentication Error, " + str(e), type=ProxyErrorTypes.auth_error, diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index ee012e65ab1..c002b2b9508 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -338,6 +338,10 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = ( # re-route the request's retention and accounting to any project # reachable with the deployment's shared AWS credentials. "aws_bedrock_project_id", + "workspace_id", + "aws_workspace_id", + "anthropic_workspace_id", + "anthropic-workspace-id", "bedrock_tags", # Provider-specific endpoint overrides that flow into the outbound # request via ``optional_params``. Same threat as ``api_base``: diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 010a1b4536e..e07b20fd5d5 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -70,6 +70,7 @@ from litellm.types.agents import AgentResponse from litellm.types.proxy.auth.auth_checks import UserNotFoundError from .auth_checks import ( + TeamNotFoundError, _allowed_routes_check, allowed_routes_check, get_actual_routes, @@ -147,6 +148,12 @@ class _JWTProvisioning: team_id_upsert: bool +@dataclass(frozen=True, slots=True) +class HeaderTeam: + header_value: str + team_id: str + + class AgentLookup(Protocol): """The registered-agent lookups a JWT agent claim is matched against.""" @@ -1871,48 +1878,108 @@ class JWTAuthManager: return True @staticmethod - def get_team_id_from_header( - request_headers: Mapping[str, str] | None, - allowed_team_ids: set[str], - fallback_to_db_teams: bool = False, - ) -> str | None: - """ - Extract team_id from x-litellm-team-id header if present. - Validates that the team is in the user's allowed teams from JWT. - - Args: - request_headers: Dictionary of request headers - allowed_team_ids: Set of team IDs the user is allowed to access (from JWT) - fallback_to_db_teams: When True and the JWT carries no team claims - (allowed_team_ids is empty), the header value is returned - provisionally and validated against DB memberships later in - auth_builder instead of being rejected here. - - Returns: - The team_id from header if valid, None otherwise - - Raises: - HTTPException: If team_id is provided but not in allowed_team_ids - """ + def _team_header_value(request_headers: Mapping[str, str] | None) -> str | None: if not request_headers: return None - - # Normalize headers to lowercase for case-insensitive lookup normalized_headers: Final = {k.lower(): v for k, v in request_headers.items()} - header_team_id: Final = normalized_headers.get("x-litellm-team-id") + return normalized_headers.get("x-litellm-team-id") - if not header_team_id: + @staticmethod + def _raise_header_team_not_allowed(header_value: str, allowed_team_ids: set[str]) -> NoReturn: + raise HTTPException( + status_code=403, + detail=( + f"x-litellm-team-id '{header_value}' does not resolve to a team id or a unique team alias in your " + f"JWT's allowed teams. Allowed team ids: {sorted(allowed_team_ids)}" + ), + ) + + @staticmethod + async def _team_id_by_alias( + team_alias: str, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, + parent_otel_span: Span | None, + proxy_logging_obj: ProxyLogging, + ) -> str | None: + if prisma_client is None: + return None + try: + team: Final = await get_team_object_by_alias( + team_alias=team_alias, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + except HTTPException as exc: + if exc.status_code >= 500: + raise + return None + return team.team_id + + @staticmethod + async def resolve_team_from_header( + request_headers: Mapping[str, str] | None, + allowed_team_ids: set[str], + fallback_to_db_teams: bool, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, + parent_otel_span: Span | None, + proxy_logging_obj: ProxyLogging, + ) -> HeaderTeam | None: + """ + The team named by x-litellm-team-id, which may carry a team id or a team + alias. A value that is already an allowed team id (or, under the DB + fallback, an existing team id) never costs an alias lookup; an alias is + accepted only when the team it names would have been accepted by id. + Under the DB fallback only a team row that is provably absent falls + through to the alias lookup; a read that failed for any other reason + keeps the membership denial the id path already gives. + + Raises: + HTTPException: 403 when neither the value nor the team it aliases is + an allowed team, or the DB fallback's membership denial when the + value names no team at all; an alias several teams share resolves + to no team and is denied like an unknown value; a 5xx from the + alias lookup itself is re-raised rather than reported as a denial + """ + header_value: Final = JWTAuthManager._team_header_value(request_headers) + if not header_value: return None - defer_to_db_membership: Final = fallback_to_db_teams and not allowed_team_ids - if not defer_to_db_membership and header_team_id not in allowed_team_ids: - raise HTTPException( - status_code=403, - detail=f"Team '{header_team_id}' from x-litellm-team-id header is not in your JWT's allowed teams. Allowed teams: {list(allowed_team_ids)}", - ) + if fallback_to_db_teams and not allowed_team_ids: + try: + await get_team_object( + team_id=header_value, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + team_id_upsert=False, + ) + except TeamNotFoundError: + aliased_team_id: Final = await JWTAuthManager._team_id_by_alias( + header_value, prisma_client, user_api_key_cache, parent_otel_span, proxy_logging_obj + ) + if aliased_team_id is None: + JWTAuthManager._raise_header_team_membership_denial(header_value) + return HeaderTeam(header_value=header_value, team_id=aliased_team_id) + except HTTPException: + JWTAuthManager._raise_header_team_membership_denial(header_value) + return HeaderTeam(header_value=header_value, team_id=header_value) - verbose_proxy_logger.debug("Using team_id from x-litellm-team-id header: %s", header_team_id) - return header_team_id + if header_value in allowed_team_ids: + verbose_proxy_logger.debug("Using team_id from x-litellm-team-id header: %s", header_value) + return HeaderTeam(header_value=header_value, team_id=header_value) + + team_id_by_alias: Final = await JWTAuthManager._team_id_by_alias( + header_value, prisma_client, user_api_key_cache, parent_otel_span, proxy_logging_obj + ) + if team_id_by_alias is None or team_id_by_alias not in allowed_team_ids: + JWTAuthManager._raise_header_team_not_allowed(header_value, allowed_team_ids) + verbose_proxy_logger.debug("Using team_id %s for x-litellm-team-id alias: %s", team_id_by_alias, header_value) + return HeaderTeam(header_value=header_value, team_id=team_id_by_alias) @staticmethod async def map_user_to_teams( @@ -2099,8 +2166,11 @@ class JWTAuthManager: spend / metadata can be attributed correctly. Returns (team_id, team_object, team_membership_object). - Any DB error is debug-logged and the tuple is (None, None, None) — no - exception ever propagates from this helper. + A team that cannot be loaded (HTTPException from get_team_object) is + debug-logged and the tuple is (None, None, None), the same as the DB + team fallback. A failed membership read propagates, so a database + outage surfaces as the 503 the rest of auth answers with instead of + serving the request with the member's limits dropped. """ if user_object is None or not user_object.teams or len(user_object.teams) != 1: return None, None, None @@ -2115,28 +2185,28 @@ class JWTAuthManager: proxy_logging_obj=proxy_logging_obj, team_id_upsert=team_id_upsert, ) - if team_row is None: - return None, None, None - - if not user_id: - return _tid, team_row, None - - team_membership: Final = await get_team_membership( - user_id=user_id, - team_id=_tid, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - ) - return _tid, team_row, team_membership - except Exception: + except HTTPException: verbose_proxy_logger.debug( - "JWT single-team fallback error, skipping. team_id=%s", + "JWT single-team fallback: team could not be loaded, skipping. team_id=%s", _tid, exc_info=True, ) return None, None, None + if team_row is None: + return None, None, None + + if not user_id: + return _tid, team_row, None + + team_membership: Final = await get_team_membership( + user_id=user_id, + team_id=_tid, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + return _tid, team_row, team_membership @staticmethod async def _resolve_db_team_fallback( @@ -2261,31 +2331,37 @@ class JWTAuthManager: ) @staticmethod - def _raise_header_team_membership_denial(team_id: str) -> NoReturn: + def _raise_header_team_membership_denial(header_value: str) -> NoReturn: """ The single denial shape for a provisional x-litellm-team-id header, raised identically for nonexistent teams and for teams the user is not - a member of, so the response does not reveal whether a team id exists. + a member of, and naming only the value the caller sent, so the response + reveals neither whether a team exists nor which id an alias maps to. """ raise HTTPException( status_code=403, - detail=(f"Team '{team_id}' (from x-litellm-team-id header) is not in your team memberships."), + detail=( + f"x-litellm-team-id '{header_value}' does not resolve to a team id or a unique team alias among your " + "team memberships." + ), ) @staticmethod def _validate_header_team_in_db_membership( team_id: str, user_object: LiteLLM_UserTable | None, + header_value: str, ) -> None: """ A provisional team_id from the x-litellm-team-id header (accepted without JWT-team validation when the JWT carries no team claims) must exist in the - user's DB team memberships before it becomes request context. + user's DB team memberships before it becomes request context. The denial + names `header_value`, the id or alias the caller sent, not `team_id`. """ user_team_ids: Final = user_object.teams if user_object else [] if team_id in user_team_ids: return - JWTAuthManager._raise_header_team_membership_denial(team_id) + JWTAuthManager._raise_header_team_membership_denial(header_value) @staticmethod async def auth_builder( @@ -2511,13 +2587,17 @@ class JWTAuthManager: if specific_team_id and not db_team_fallback: all_team_ids.add(specific_team_id) - header_team_id: Final = JWTAuthManager.get_team_id_from_header( + header_team: Final = await JWTAuthManager.resolve_team_from_header( request_headers=request_headers, allowed_team_ids=all_team_ids, fallback_to_db_teams=db_team_fallback, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, ) - if header_team_id: - team_id = header_team_id + if header_team: + team_id = header_team.team_id # A provisional header team (accepted only because the JWT carries no # team claims) is validated against DB membership further down; never # upsert it here or an attacker-supplied x-litellm-team-id would create @@ -2535,7 +2615,7 @@ class JWTAuthManager: except HTTPException: if not db_team_fallback: raise - JWTAuthManager._raise_header_team_membership_denial(team_id) + JWTAuthManager._raise_header_team_membership_denial(header_team.header_value) elif not team_id and not db_team_fallback: ## SPECIFIC TEAM ID ( @@ -2676,10 +2756,11 @@ class JWTAuthManager: proxy_logging_obj=proxy_logging_obj, team_id_upsert=team_id_upsert, ) - elif db_team_fallback and team_id == header_team_id: + elif db_team_fallback and header_team is not None and team_id == header_team.team_id: JWTAuthManager._validate_header_team_in_db_membership( team_id=team_id, user_object=user_object, + header_value=header_team.header_value, ) if not JWTAuthManager._is_team_route_allowed( route=route, @@ -2689,7 +2770,8 @@ class JWTAuthManager: raise HTTPException( status_code=403, detail=( - f"Team '{team_id}' (from x-litellm-team-id header) is not allowed to access route '{route}'." + f"Team '{header_team.header_value}' (from x-litellm-team-id header) " + f"is not allowed to access route '{route}'." ), ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 08f64e610bd..a6c0792a86f 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -650,6 +650,7 @@ async def user_api_key_auth_websocket_for_model(websocket: WebSocket, model: str "type": "http", "headers": scope_headers, "path": ws_scope.get("path", ""), + "state": ws_scope.setdefault("state", {}), # mutable-ok: Starlette's socket state, shared with the request } for key in ("root_path", "app_root_path"): if key in ws_scope: @@ -3086,31 +3087,30 @@ async def _reserve_budget_after_common_checks( request: Request | None = None, ) -> None: user_api_key_auth_obj.budget_reservation = None - if skip_budget_checks: - return - if general_settings.get("disable_budget_reservation") is True: - return + if not skip_budget_checks and general_settings.get("disable_budget_reservation") is not True: + from litellm.proxy.spend_tracking.budget_reservation import ( + reserve_budget_for_request, + ) - from litellm.proxy.spend_tracking.budget_reservation import ( - reserve_budget_for_request, - ) - - user_api_key_auth_obj.budget_reservation = await reserve_budget_for_request( - request_body=request_data, - route=route, - llm_router=llm_router, - valid_token=user_api_key_auth_obj, - team_object=team_object, - user_object=user_object, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - end_user_id=end_user_id, - end_user_object=end_user_object, - apply_user_budget_to_team_keys=general_settings.get("apply_user_budget_to_team_keys") is True, - fail_closed_budget_enforcement=general_settings.get("fail_closed_budget_enforcement") is True, - raw_body=await read_raw_json_body(request=request), - ) + user_api_key_auth_obj.budget_reservation = await reserve_budget_for_request( + request_body=request_data, + route=route, + llm_router=llm_router, + valid_token=user_api_key_auth_obj, + team_object=team_object, + user_object=user_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + end_user_id=end_user_id, + end_user_object=end_user_object, + apply_user_budget_to_team_keys=general_settings.get("apply_user_budget_to_team_keys") is True, + fail_closed_budget_enforcement=general_settings.get("fail_closed_budget_enforcement") is True, + raw_body=await read_raw_json_body(request=request), + ) + if request is not None: + reservation: Final = user_api_key_auth_obj.budget_reservation + request.state.budget_reservation = reservation # rebind-ok: read by the release middleware def _should_skip_budget_checks( diff --git a/litellm/proxy/bug_report_config.py b/litellm/proxy/bug_report_config.py new file mode 100644 index 00000000000..d7b920de4d5 --- /dev/null +++ b/litellm/proxy/bug_report_config.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import ast +import functools +import inspect +from collections.abc import Mapping, Sequence +from pathlib import Path +from types import MappingProxyType +from typing import Final + +from pydantic import JsonValue, TypeAdapter, ValidationError + +import litellm +from litellm.litellm_core_utils.bug_report import ( + KNOWN_PROVIDERS, + BugReport, + EnvironmentReport, + allowlisted, + build_bug_report, + build_environment_report, +) +from litellm.proxy._types import ConfigGeneralSettings +from litellm.router_utils.routing_groups import VALID_ROUTING_STRATEGIES +from litellm.types.caching import LiteLLMCacheType +from litellm.types.guardrails import GuardrailEventHooks, LitellmParams, SupportedGuardrailIntegrations +from litellm.types.secret_managers.main import KeyManagementSystem + +_OBJECT_MAP: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_OBJECT_LIST: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +CREDENTIAL_KEY_PARTS: Final = frozenset( + { + "key", + "keys", + "secret", + "secrets", + "token", + "password", + "passwd", + "credential", + "credentials", + "url", + "uri", + "dsn", + "host", + "hosts", + "base", + "endpoint", + "cert", + "pem", + "salt", + } +) +ENUM_KEYS_WITH_CREDENTIAL_PARTS: Final = frozenset({"key_management_system"}) + + +def _object_map(value: object) -> Mapping[str, object]: + try: + return _OBJECT_MAP.validate_python(value) + except ValidationError: + return MappingProxyType({}) + + +def _object_list(value: object) -> Sequence[object]: + try: + return _OBJECT_LIST.validate_python(value) + except ValidationError: + return () + + +@functools.cache +def _known_values() -> frozenset[str]: + from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry + + return frozenset( + ( + *VALID_ROUTING_STRATEGIES, + *KNOWN_PROVIDERS, + *litellm._known_custom_logger_compatible_callbacks, # pyright: ignore[reportPrivateUsage, reportUnknownMemberType, reportUnknownArgumentType] # untyped List of the callback Literal's args, no public alias + *CustomLoggerRegistry.CALLBACK_CLASS_STR_TO_CLASS_TYPE, + *(member.value for member in LiteLLMCacheType), + *(member.value for member in KeyManagementSystem), + *(member.value for member in SupportedGuardrailIntegrations), + *(member.value for member in GuardrailEventHooks), + ) + ) + + +def _module_level_names(node: ast.stmt) -> tuple[str, ...]: + match node: + case ast.Assign(targets=targets): + return tuple(target.id for target in targets if isinstance(target, ast.Name)) + case ast.AnnAssign(target=ast.Name(id=name)): + return (name,) + case ast.ImportFrom(names=aliases): + return tuple(alias.asname or alias.name for alias in aliases) + case _: + return () + + +@functools.cache +def _litellm_settings_keys() -> frozenset[str]: + tree: Final = ast.parse(Path(litellm.__file__).read_text()) + return frozenset(name for node in tree.body for name in _module_level_names(node)) + + +@functools.cache +def _router_settings_keys() -> frozenset[str]: + from litellm.router import Router + + return frozenset(name for name in inspect.signature(Router.__init__).parameters if name != "self") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped params, only names are read + + +@functools.cache +def _cache_params_keys() -> frozenset[str]: + from litellm.caching.caching import Cache + + return frozenset(name for name in inspect.signature(Cache.__init__).parameters if name != "self") # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped params, only names are read + + +def _is_credential_key(key: str) -> bool: + return key not in ENUM_KEYS_WITH_CREDENTIAL_PARTS and not CREDENTIAL_KEY_PARTS.isdisjoint(key.lower().split("_")) + + +def _render_json(key: str, value: JsonValue) -> str | None: + match value: + case bool(): + return str(value).lower() + case str(): + return value if value in _known_values() and not _is_credential_key(key) else None + case list(): + known_items: Final = tuple(rendered for item in value if (rendered := _render_json(key, item)) is not None) + return f"[{', '.join(known_items)}]" if known_items else None + case _: + return None + + +def _render(key: str, value: object) -> str | None: + try: + return _render_json(key, _JSON.validate_python(value)) + except ValidationError: + return None + + +def _section_lines(section: str, values: Mapping[str, object], known_keys: frozenset[str]) -> tuple[str, ...]: + return tuple( + f"{section}.{key} = {rendered}" + for key, value in values.items() + if key in known_keys and (rendered := _render(key, value)) is not None + ) + + +def _guardrail_lines(guardrails: object) -> tuple[str, ...]: + known_keys: Final = frozenset(LitellmParams.model_fields) + return tuple( + line + for index, guardrail in enumerate(_object_list(guardrails)) + for line in _section_lines( + f"guardrails[{index}].litellm_params", _object_map(_object_map(guardrail).get("litellm_params")), known_keys + ) + ) + + +def _deployment_provider(model: object) -> str | None: + prefix: Final = model.split("/", 1)[0] if isinstance(model, str) and "/" in model else None + return allowlisted(prefix, KNOWN_PROVIDERS) + + +def _model_list_lines(model_list: object) -> tuple[str, ...]: + providers: Final = tuple( + sorted( + frozenset( + provider + for deployment in _object_list(model_list) + if ( + provider := _deployment_provider( + _object_map(_object_map(deployment).get("litellm_params")).get("model") + ) + ) + is not None + ) + ) + ) + return (f"model_list[*].provider = [{', '.join(providers)}]",) if providers else () + + +def safe_config_lines(config: Mapping[str, object], general_settings: Mapping[str, object]) -> tuple[str, ...]: + litellm_settings: Final = _object_map(config.get("litellm_settings")) + return ( + *_section_lines("general_settings", general_settings, frozenset(ConfigGeneralSettings.model_fields)), + *_section_lines("litellm_settings", litellm_settings, _litellm_settings_keys()), + *_section_lines( + "litellm_settings.cache_params", _object_map(litellm_settings.get("cache_params")), _cache_params_keys() + ), + *_section_lines("router_settings", _object_map(config.get("router_settings")), _router_settings_keys()), + *_guardrail_lines(config.get("guardrails")), + *_model_list_lines(config.get("model_list")), + ) + + +def _proxy_config_lines() -> tuple[str, ...]: + from litellm.proxy import proxy_server + + return safe_config_lines( + proxy_server.proxy_config.config, + _object_map(proxy_server.general_settings), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # bare dict global, validated by _object_map + ) + + +def build_proxy_environment_report() -> EnvironmentReport: + return build_environment_report(surface="proxy", config_lines=_proxy_config_lines()) + + +def build_proxy_bug_report( + exc: BaseException, + *, + call_type: str | None = None, + custom_llm_provider: object = None, + stream: object = None, +) -> BugReport: + return build_bug_report( + exc, + surface="proxy", + call_type=call_type, + custom_llm_provider=custom_llm_provider, + stream=stream, + config_lines=_proxy_config_lines(), + ) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9484fd7c723..c40090233be 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -19,13 +19,14 @@ from typing import ( overload, runtime_checkable, ) +from urllib.parse import urlparse import anyio import httpx import orjson from fastapi import HTTPException, Request, status from fastapi.responses import JSONResponse, Response, StreamingResponse -from pydantic import TypeAdapter, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError from starlette.types import Receive, Scope, Send import litellm @@ -45,10 +46,17 @@ from litellm.constants import ( UNSAFE_PROXY_RESPONSE_HEADERS, ) from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.bug_report import ( + allowlisted, + bug_report_notice, + should_report_bug, + strip_bug_report_notice, +) from litellm.litellm_core_utils.core_helpers import ( get_or_create_metadata_bucket, independent_snapshot, is_expected_client_error, + redact_nested_match_and_regex_keys, ) from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer from litellm.litellm_core_utils.get_supported_openai_params import ( @@ -61,25 +69,32 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.litellm_core_utils.served_output_texts import ( + record_served_output_texts, + served_output_texts, +) from litellm.litellm_core_utils.streaming_handler import ( backfill_missing_cache_usage_fields, ) -from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth +from litellm.proxy._types import LiteLLMRoutes, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_checks import ( can_key_call_resolved_model, request_skips_budget_checks, tag_max_budget_check_for_tags, ) from litellm.proxy.auth.auth_utils import check_response_size_is_safe, get_request_route +from litellm.proxy.bug_report_config import build_proxy_bug_report from litellm.proxy.common_utils.callback_utils import ( get_logging_caching_headers, get_remaining_tokens_and_requests_from_request_data, ) +from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id from litellm.proxy.common_utils.http_parsing_utils import ( get_client_requested_model, get_tags_from_request_body, ) from litellm.proxy.common_utils.openai_error_payload import ( + LITELLM_CALL_ID_HEADER, attribute_of, error_status_code, openai_error_param, @@ -93,6 +108,7 @@ from litellm.proxy.common_utils.sse_keepalive import ( ) from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression +from litellm.proxy.native_compaction import with_proxy_compaction_executor from litellm.proxy.route_llm_request import route_request from litellm.proxy.utils import ProxyLogging, _check_and_merge_model_level_guardrails from litellm.router import Router @@ -105,6 +121,10 @@ from litellm.types.router_weights import validate_router_weights _LateResponseT = TypeVar("_LateResponseT", bound=Response) _LlmCallT = TypeVar("_LlmCallT") +KNOWN_PROXY_ROUTES: Final = frozenset( + route for member in LiteLLMRoutes for route in member.value if route.startswith("/") +) + ProxyRouteType: TypeAlias = Literal[ "acompletion", "aembedding", @@ -946,6 +966,9 @@ async def _resolve_stream_headers( return headers +_NO_GENERAL_SETTINGS: Final[Mapping[str, object]] = MappingProxyType({}) + + async def create_response( generator: AsyncGenerator[str, None], media_type: str, @@ -953,6 +976,7 @@ async def create_response( default_status_code: int = status.HTTP_200_OK, request: Request | None = None, refresh_headers: Callable[[], Awaitable[Mapping[str, str]]] | None = None, + general_settings: Mapping[str, object] = _NO_GENERAL_SETTINGS, ) -> StreamingResponse | JSONResponse: """ Create streaming response, checking if the first chunk is an error. @@ -960,7 +984,8 @@ async def create_response( Otherwise, return StreamingResponse and stream all content. ``refresh_headers`` is consulted once the first chunk has been buffered, for - callers whose headers can only be known then. + callers whose headers can only be known then. ``general_settings`` decides whether + the first-chunk error body also carries the ``x-litellm-call-id`` header's value. """ first_chunk_value: str | None = None final_status_code = default_status_code @@ -987,7 +1012,10 @@ async def create_response( ) # Parse error content - error_dict: Final = _extract_error_from_sse_chunk(first_chunk_value) + error_dict: Final = with_call_id( + JSON_OBJECT.validate_python(_extract_error_from_sse_chunk(first_chunk_value)), + error_body_call_id(general_settings, resolved_headers.get(LITELLM_CALL_ID_HEADER)), + ) # Consume and close generator (avoid resource leak) try: @@ -1374,6 +1402,42 @@ def _override_openai_response_model( ) +_METADATA_BUCKET_KEYS: Final = ("metadata", "litellm_metadata") +_RESPONSE_REDACTED_KEYS: Final = ("keyword", "snippet", "match", "regex") + + +def _request_metadata_buckets(request_data: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + return tuple(bucket for key in _METADATA_BUCKET_KEYS if isinstance(bucket := request_data.get(key), Mapping)) + + +def include_guardrail_response_requested(request_data: Mapping[str, object]) -> bool: + return any(bucket.get("include_guardrail_response") is True for bucket in _request_metadata_buckets(request_data)) + + +def attach_guardrail_information(response: object, request_data: Mapping[str, object]) -> object: + recorded: Final[Sequence[object]] = next( + ( + entries + for bucket in _request_metadata_buckets(request_data) + if isinstance( + entries := bucket.get("standard_logging_guardrail_information"), + list, + ) + ), + (), + ) + guardrail_information: Final = [ # mutable-ok: response list contract + redact_nested_match_and_regex_keys(entry, keys=_RESPONSE_REDACTED_KEYS) + for entry in recorded + if isinstance(entry, dict) + ] + if isinstance(response, dict): + return response | MappingProxyType({"guardrail_information": guardrail_information}) + if isinstance(response, BaseModel) and response.model_config.get("extra") == "allow": + return response.model_copy(update=MappingProxyType({"guardrail_information": guardrail_information})) + return response + + class CostBreakdownHeaderValues(NamedTuple): original_cost: float | None = None discount_amount: float | None = None @@ -2533,7 +2597,7 @@ class ProxyBaseLLMRequestProcessing: user_model=user_model, user_api_key_dict=user_api_key_dict, ) - llm_call_task: Final = asyncio.create_task(llm_call) + llm_call_task: Final = asyncio.create_task(with_proxy_compaction_executor(llm_call, request)) tasks.append(llm_call_task) llm_responses: Final = asyncio.gather(*tasks) # run the moderation check in parallel to the actual llm api call @@ -2738,6 +2802,7 @@ class ProxyBaseLLMRequestProcessing: headers=custom_headers, request=request, refresh_headers=refresh_stream_headers, + general_settings=general_settings, ) ### CALL HOOKS ### - modify outgoing data @@ -2779,6 +2844,7 @@ class ProxyBaseLLMRequestProcessing: user_api_key_dict=user_api_key_dict, response=response, ) + record_served_output_texts(logging_obj.model_call_details, served_output_texts(response)) except Exception: _exception_raised = True raise @@ -2869,6 +2935,11 @@ class ProxyBaseLLMRequestProcessing: if isinstance(response, dict): response.pop("_hidden_params", None) + if include_guardrail_response_requested(self.data): + response = attach_guardrail_information( # rebind-ok: response tail rebinds the copied response + response=response, request_data=self.data + ) + # Call response headers hook for non-streaming success callback_headers = await proxy_logging_obj.post_call_response_headers_hook( data=self.data, @@ -3658,8 +3729,27 @@ class ProxyBaseLLMRequestProcessing: _code = _exc_status_code else: _code = status.HTTP_500_INTERNAL_SERVER_ERROR + if should_report_bug(e): + proxy_server_request: Final = self.data.get("proxy_server_request") + request_url: Final = ( + proxy_server_request.get("url") if isinstance(proxy_server_request, Mapping) else None + ) + request_path: Final = urlparse(str(request_url)).path if request_url is not None else None + verbose_proxy_logger.error( + bug_report_notice( + build_proxy_bug_report( + e, + call_type=allowlisted(request_path, KNOWN_PROXY_ROUTES), + custom_llm_provider=self.data.get("custom_llm_provider"), + stream=self.data.get("stream"), + ) + ) + ) + client_message: Final = getattr(e, "message", error_msg) raise ProxyException( - message=redact_internal_details_from_client_message(getattr(e, "message", error_msg)), + message=redact_internal_details_from_client_message( + strip_bug_report_notice(client_message) if isinstance(client_message, str) else error_msg + ), type=openai_error_type(e, _code), param=openai_error_param(e), openai_code=getattr(e, "code", None), diff --git a/litellm/proxy/common_utils/callback_config_validation.py b/litellm/proxy/common_utils/callback_config_validation.py index 30c4ab31d6f..6a0fcbe0bb3 100644 --- a/litellm/proxy/common_utils/callback_config_validation.py +++ b/litellm/proxy/common_utils/callback_config_validation.py @@ -5,6 +5,7 @@ Team callbacks arrive as a single ``AddTeamCallback``, key callbacks arrive as a per-integration checks here. """ +import math from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import Final @@ -13,11 +14,18 @@ _NEWRELIC_CALLBACK: Final = "newrelic" _NEWRELIC_VAR_PREFIX: Final = "newrelic_" _LANGFUSE_OTEL_CALLBACK: Final = "langfuse_otel" _LANGFUSE_SPAN_SCOPE_VAR: Final = "langfuse_span_scope" +_ARIZE_CALLBACK: Final = "arize" +_ARIZE_SAMPLING_RATE_VARS: Final[frozenset[str]] = frozenset( + {"arize_success_sampling_rate", "arize_error_sampling_rate"} +) def callback_config_error(callback_name: str | None, callback_vars: Mapping[str, str] | None) -> str | None: if not callback_vars: return None + arize_error: Final = _arize_sampling_rate_error(callback_name, callback_vars) + if arize_error is not None: + return arize_error langfuse_error: Final = _langfuse_environment_error(callback_vars) or _langfuse_span_scope_error( callback_name, callback_vars ) @@ -86,7 +94,7 @@ _VAR_FAMILIES: Final[Mapping[str, str]] = MappingProxyType( } ) -_FAMILY_OPTION_VARS: Final[frozenset[str]] = frozenset({_LANGFUSE_SPAN_SCOPE_VAR}) +_FAMILY_OPTION_VARS: Final[frozenset[str]] = frozenset({_LANGFUSE_SPAN_SCOPE_VAR, *_ARIZE_SAMPLING_RATE_VARS}) def _family_of(var: str) -> str | None: @@ -212,6 +220,22 @@ def _logging_entry_error(entry: object) -> str | None: return callback_config_error(callback_name, _entry_callback_vars(entry)) +def _arize_sampling_rate_error(callback_name: str | None, callback_vars: Mapping[str, str]) -> str | None: + for var in sorted(_ARIZE_SAMPLING_RATE_VARS): + value = callback_vars.get(var) + if value is None or value in ("", "None"): + continue + if callback_name != _ARIZE_CALLBACK: + return f"{var} applies to the {_ARIZE_CALLBACK} callback only, not {callback_name!r}" + try: + rate = float(value) + except (TypeError, ValueError): + return f"{var} must be a number between 0.0 and 1.0 (inclusive), got {value!r}" + if not math.isfinite(rate) or not 0.0 <= rate <= 1.0: + return f"{var} must be a number between 0.0 and 1.0 (inclusive), got {value!r}" + return None + + def _newrelic_config_error(callback_vars: Mapping[str, str]) -> str | None: """Per-team New Relic routing runs on the OTel v2 path only. diff --git a/litellm/proxy/common_utils/config_sync_pubsub.py b/litellm/proxy/common_utils/config_sync_pubsub.py index 6d781babe63..b20c0d9c9a5 100644 --- a/litellm/proxy/common_utils/config_sync_pubsub.py +++ b/litellm/proxy/common_utils/config_sync_pubsub.py @@ -55,6 +55,7 @@ _CONFIG_SYNCED_TABLE_NAMES: Final[frozenset[str]] = frozenset( "litellm_ssoconfig", "litellm_cacheconfig", "litellm_configoverrides", + "litellm_uisettings", } ) diff --git a/litellm/proxy/common_utils/debug_utils.py b/litellm/proxy/common_utils/debug_utils.py index dc329e55e31..2544321a1b6 100644 --- a/litellm/proxy/common_utils/debug_utils.py +++ b/litellm/proxy/common_utils/debug_utils.py @@ -8,7 +8,7 @@ import sys import tracemalloc from collections import Counter from collections.abc import Mapping, Sequence -from typing import Any, Final, NamedTuple, Protocol, TypedDict +from typing import Annotated, Any, Final, NamedTuple, Protocol, TypedDict from fastapi import APIRouter, Depends, HTTPException, Query from typing_extensions import ReadOnly @@ -16,8 +16,11 @@ from typing_extensions import ReadOnly from litellm import get_secret_str from litellm._logging import verbose_proxy_logger from litellm.constants import PYTHON_GC_THRESHOLD +from litellm.litellm_core_utils.bug_report import EnvironmentReport from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.bug_report_config import build_proxy_environment_report +from litellm.proxy.common_utils.resource_ownership import is_proxy_admin router: Final = APIRouter() @@ -783,6 +786,23 @@ async def configure_gc_thresholds_endpoint( } +@router.get("/debug/report", include_in_schema=False) +async def get_debug_report( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> EnvironmentReport: + """ + The same LiteLLM-owned environment facts the bug report link puts in a GitHub issue: + versions, deployment kind, and config flags whose keys and values LiteLLM defines. + Nothing from the operator's config values, request data, or errors + + Example usage: + curl http://localhost:4000/debug/report -H "Authorization: Bearer sk-1234" + """ + if not is_proxy_admin(user_api_key_dict): + raise HTTPException(status_code=403, detail="Only proxy admins can read /debug/report") + return build_proxy_environment_report() + + @router.get( "/otel-spans", dependencies=[Depends(user_api_key_auth)], diff --git a/litellm/proxy/common_utils/error_body_call_id.py b/litellm/proxy/common_utils/error_body_call_id.py new file mode 100644 index 00000000000..f50be5df509 --- /dev/null +++ b/litellm/proxy/common_utils/error_body_call_id.py @@ -0,0 +1,20 @@ +from collections.abc import Mapping +from typing import Final + +from pydantic import TypeAdapter + +INCLUDE_CALL_ID_IN_ERROR_BODY_SETTING: Final = "include_call_id_in_error_body" +LITELLM_CALL_ID_BODY_KEY: Final = "litellm_call_id" +JSON_OBJECT: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) # mutable-ok: JSONResponse input + + +def error_body_call_id(general_settings: Mapping[str, object], call_id: str | None) -> str | None: + if general_settings.get(INCLUDE_CALL_ID_IN_ERROR_BODY_SETTING) is not True: + return None + return call_id if call_id else None + + +def with_call_id(error: dict[str, object], call_id: str | None) -> dict[str, object]: # mutable-ok: JSONResponse input + if call_id is None: + return error + return {**error, LITELLM_CALL_ID_BODY_KEY: call_id} # mutable-ok: JSONResponse input diff --git a/litellm/proxy/common_utils/swagger_utils.py b/litellm/proxy/common_utils/swagger_utils.py index 2609a98a997..83480bf161d 100644 --- a/litellm/proxy/common_utils/swagger_utils.py +++ b/litellm/proxy/common_utils/swagger_utils.py @@ -1,3 +1,4 @@ +import inspect from typing import Any, Final from pydantic import BaseModel, Field @@ -31,11 +32,15 @@ def get_status_code(exception): return 500 # Internal Server Error as default +def _error_description(exception: type[Exception]) -> str: + return inspect.cleandoc(exception.__doc__) if exception.__doc__ else exception.__name__ + + # Create error responses ERROR_RESPONSES: Final = { get_status_code(exception): { "model": ErrorResponse, - "description": exception.__doc__ or exception.__name__, + "description": _error_description(exception), } for exception in LITELLM_EXCEPTION_TYPES } diff --git a/litellm/proxy/config_resolvers/settings_rules.py b/litellm/proxy/config_resolvers/settings_rules.py index f346dd6198d..1f0adfc5248 100644 --- a/litellm/proxy/config_resolvers/settings_rules.py +++ b/litellm/proxy/config_resolvers/settings_rules.py @@ -48,6 +48,7 @@ _UI_SETTINGS_FIELDS: Final[tuple[str, ...]] = ( "allow_agents_for_team_admins", "disable_vector_stores_for_internal_users", "allow_vector_stores_for_team_admins", + "disable_custom_api_keys", "disable_key_generate_for_org_admin", "team_admin_editable_team_fields", ) diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index f25c2787252..38d6fb9b99d 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -1,5 +1,6 @@ import re from collections.abc import Awaitable, Callable, Iterator +from http import HTTPStatus from typing import Final, Protocol, TypeVar from pydantic import TypeAdapter, ValidationError @@ -378,6 +379,15 @@ class PrismaDBExceptionHandler: "The proxy deployment needs attention." ) + @staticmethod + def service_unavailable_proxy_exception(e: Exception) -> ProxyException: + return ProxyException( + message=PrismaDBExceptionHandler.database_unavailable_message(e), + type=ProxyErrorTypes.no_db_connection, + param="None", + code=HTTPStatus.SERVICE_UNAVAILABLE.value, + ) + @staticmethod def find_database_service_unavailable_error_in_chain(e: BaseException) -> Exception | None: """The exception in the ``__cause__`` / ``__context__`` chain that @@ -398,11 +408,8 @@ class PrismaDBExceptionHandler: ``is_database_service_unavailable_error`` classifies a single exception by type, which a caller that catches a raw DB failure and re-raises a - domain exception of a different type defeats. ``get_user_object`` in - ``litellm/proxy/auth/auth_checks.py`` is the concrete case: it wraps - every DB error, a genuine outage included, in a bare ``ValueError`` - whose original error survives only as ``__context__``. A type check on - the ``ValueError`` misses the outage, so the caller would mistake an + domain exception of a different type defeats. A type check on the + wrapper misses the outage, so the caller would mistake an infrastructure fault for an auth failure. Walking the chain recovers the real signal, which is the PEP 3134 way to inspect a wrapped cause. diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 70ea21320ee..fe91d6d7a28 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -11,7 +11,7 @@ import asyncio import json import threading -from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Sequence +from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, Sequence from contextlib import asynccontextmanager from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypedDict, cast @@ -39,6 +39,11 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.guardrails.anthropic_sse import ( + anthropic_sse_chunks_from_response, + assemble_anthropic_sse_stream, + model_response_text, +) from litellm.types.guardrails import ( GuardrailEventHooks, LitellmParams, @@ -1327,30 +1332,44 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): ) return response - async def _stream_apply_output_masking( - self, - response: AsyncIterable[object], - request_data: dict, - ) -> AsyncGenerator[ModelResponseStream | bytes, None]: - """Apply Presidio masking to streaming output (apply_to_output=True path).""" + async def _mask_buffered_model_response_stream( + self, all_chunks: Sequence[ModelResponseStream], request_data: dict + ) -> tuple[object, ...]: from litellm.llms.base_llm.base_model_iterator import ( convert_model_response_to_streaming, ) from litellm.main import stream_chunk_builder from litellm.types.utils import ModelResponse + assembled: Final = stream_chunk_builder(chunks=list(all_chunks), messages=request_data.get("messages")) + if not isinstance(assembled, ModelResponse): + return tuple(all_chunks) + await self._process_response_for_pii(response=assembled, request_data=request_data, mode="mask") + return (convert_model_response_to_streaming(assembled),) + + async def _stream_apply_output_masking( + self, + response: AsyncIterable[object], + request_data: dict, + ) -> AsyncGenerator[object, None]: + """Apply Presidio masking to streaming output (apply_to_output=True path).""" all_chunks: list[ModelResponseStream] = [] passthrough_due_to_unknown_stream_shape = False try: - async for chunk in response: + stream: Final = response.__aiter__() + async for chunk in stream: if isinstance(chunk, ModelResponseStream): if passthrough_due_to_unknown_stream_shape: yield chunk else: all_chunks.append(chunk) elif isinstance(chunk, bytes): - yield chunk - continue + if passthrough_due_to_unknown_stream_shape or all_chunks: + yield chunk + continue + for masked_chunk in await self._mask_anthropic_sse_stream(chunk, stream, request_data): + yield masked_chunk + return else: if all_chunks: # Flush buffered chunks and switch to transparent passthrough for this stream shape. @@ -1375,33 +1394,39 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if not all_chunks: verbose_proxy_logger.warning( "Presidio apply_to_output: streaming response contained no " - "ModelResponseStream chunks (e.g. raw SSE bytes or an empty " - "upstream stream). Output PII masking was skipped for this " - "response." + "ModelResponseStream chunks (an empty upstream stream). " + "Output PII masking was skipped for this response." ) return - assembled_model_response = stream_chunk_builder(chunks=all_chunks, messages=request_data.get("messages")) - - if not isinstance(assembled_model_response, ModelResponse): - for chunk in all_chunks: - yield chunk - return - - await self._process_response_for_pii( - response=assembled_model_response, - request_data=request_data, - mode="mask", - ) - - mock_response_stream: Final = convert_model_response_to_streaming(assembled_model_response) - yield mock_response_stream + for masked_chunk in await self._mask_buffered_model_response_stream(all_chunks, request_data): + yield masked_chunk except Exception as e: + if not all_chunks or isinstance(e, BlockedPiiEntityError): + raise verbose_proxy_logger.error("Error masking streaming PII output: %s", e) for chunk in all_chunks: yield chunk + async def _mask_anthropic_sse_stream( + self, first_chunk: bytes, rest: AsyncIterator[object], request_data: dict + ) -> tuple[object, ...]: + rest_chunks: Final = [chunk async for chunk in rest] # mutable-ok: tuple() cannot consume an async iterator + chunks: Final = (first_chunk, *rest_chunks) + assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True) + if assembled is None: + verbose_proxy_logger.warning( + "Presidio apply_to_output: raw SSE stream could not be assembled into a response. " + "Output PII masking was skipped for this response." + ) + return chunks + original_text: Final = model_response_text(assembled) + await self._process_response_for_pii(response=assembled, request_data=request_data, mode="mask") + if model_response_text(assembled) == original_text: + return chunks + return anthropic_sse_chunks_from_response(assembled) + @staticmethod def _unmask_sse_bytes_chunk(chunk: bytes, pii_tokens: dict[str, str]) -> bytes: try: @@ -1460,7 +1485,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): self, response: AsyncIterable[object], request_data: dict, - ) -> AsyncGenerator[ModelResponseStream | bytes, None]: + ) -> AsyncGenerator[object, None]: """Apply PII unmasking to streaming output (output_parse_pii=True path).""" from litellm.llms.base_llm.base_model_iterator import ( convert_model_response_to_streaming, @@ -1536,7 +1561,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): user_api_key_dict: UserAPIKeyAuth, response: AsyncIterable[object], request_data: dict, - ) -> AsyncGenerator[ModelResponseStream | bytes, None]: + ) -> AsyncGenerator[object, None]: """ Process streaming response chunks to unmask PII tokens when needed. diff --git a/litellm/proxy/guardrails/guardrail_initializers.py b/litellm/proxy/guardrails/guardrail_initializers.py index 356eb7c96c6..7bafad26569 100644 --- a/litellm/proxy/guardrails/guardrail_initializers.py +++ b/litellm/proxy/guardrails/guardrail_initializers.py @@ -165,6 +165,7 @@ def initialize_presidio(litellm_params: LitellmParams, guardrail: Guardrail) -> apply_to_output=True, event_hook=GuardrailEventHooks.post_call.value, output_parse_pii=False, + mask_response_content=True, ) if run_output else None diff --git a/litellm/proxy/hooks/key_management_event_hooks.py b/litellm/proxy/hooks/key_management_event_hooks.py index f75197532b4..5dc0f659a8d 100644 --- a/litellm/proxy/hooks/key_management_event_hooks.py +++ b/litellm/proxy/hooks/key_management_event_hooks.py @@ -1,7 +1,8 @@ import asyncio import json +from collections.abc import Sequence from datetime import datetime, timezone -from typing import Final +from typing import TYPE_CHECKING, Final from pydantic import TypeAdapter @@ -23,6 +24,9 @@ from litellm.proxy._types import ( from litellm.proxy.utils import _hash_token_if_needed from litellm.secret_managers.base_secret_manager import BaseSecretManager +if TYPE_CHECKING: + from prisma import models as prisma_models + # NOTE: This is the prefix for all virtual keys stored in AWS Secrets Manager LITELLM_PREFIX_STORED_VIRTUAL_KEYS: Final = "litellm/" @@ -233,6 +237,19 @@ class KeyManagementEventHooks: Handles the following: - Storing Audit Logs for key deletion """ + KeyManagementEventHooks.create_key_deleted_audit_logs( + keys_being_deleted=keys_being_deleted, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) + await KeyManagementEventHooks._delete_virtual_keys_from_secret_manager(keys_being_deleted=keys_being_deleted) + + @staticmethod + def create_key_deleted_audit_logs( + keys_being_deleted: Sequence["LiteLLM_VerificationToken | prisma_models.LiteLLM_VerificationToken"], + user_api_key_dict: UserAPIKeyAuth, + litellm_changed_by: str | None = None, + ) -> None: from litellm.proxy.management_helpers.audit_logs import ( create_audit_log_for_update, get_audit_log_changed_by, @@ -240,35 +257,33 @@ class KeyManagementEventHooks: ) from litellm.proxy.proxy_server import litellm_proxy_admin_name - # we do this after the first for loop, since first for loop is for validation. we only want this inserted after validation passes - if is_audit_logging_enabled() and data.keys is not None: - # make an audit log for each key deleted - for key in keys_being_deleted: - if key.token is None: - continue - _key_row = key.model_dump_json(exclude_none=True) + if not is_audit_logging_enabled(): + return + for key in keys_being_deleted: + key_row = LiteLLM_VerificationToken.model_validate(key, from_attributes=True) + if key_row.token is None: + continue + _key_row = key_row.model_dump_json(exclude_none=True) - asyncio.create_task( - create_audit_log_for_update( - request_data=LiteLLM_AuditLogs( - id=str(uuid.uuid4()), - updated_at=datetime.now(timezone.utc), - changed_by=get_audit_log_changed_by( - litellm_changed_by=litellm_changed_by, - user_api_key_dict=user_api_key_dict, - litellm_proxy_admin_name=litellm_proxy_admin_name, - ), - changed_by_api_key=user_api_key_dict.token, - table_name=LitellmTableNames.KEY_TABLE_NAME, - object_id=key.token, - action="deleted", - updated_values="{}", - before_value=_key_row, - ) + asyncio.create_task( + create_audit_log_for_update( + request_data=LiteLLM_AuditLogs( + id=str(uuid.uuid4()), + updated_at=datetime.now(timezone.utc), + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), + changed_by_api_key=user_api_key_dict.token, + table_name=LitellmTableNames.KEY_TABLE_NAME, + object_id=key_row.token, + action="deleted", + updated_values="{}", + before_value=_key_row, ) ) - # delete the keys from the secret manager - await KeyManagementEventHooks._delete_virtual_keys_from_secret_manager(keys_being_deleted=keys_being_deleted) + ) @staticmethod async def _store_virtual_key_in_secret_manager(secret_name: str, secret_token: str, team_id: str | None = None): diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index a6b00be1091..33744906b13 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -63,6 +63,7 @@ from litellm.router_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, response_has_hidden_params, ) +from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage from litellm.types.utils import ( @@ -91,6 +92,26 @@ else: _REQUEST_RATE_LIMIT_DATA: Final = TypeAdapter(Mapping[str, object]) +@dataclass(frozen=True, slots=True) +class RateLimitedModel: + requested: str + group: str + + def limit_in(self, limits: Mapping[str, int] | None) -> int | None: + if limits is None: + return None + requested_limit: Final = limits.get(self.requested) + return requested_limit if requested_limit is not None else limits.get(self.group) + + +def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None: + from litellm.proxy.proxy_server import llm_router + + if llm_router is None: + return None + return resolve_model_group_alias(llm_router.model_group_alias, model) + + def _sibling_counter_keys(window_key: str) -> tuple[str, str]: prefix: Final = window_key.removesuffix(":window") return f"{prefix}:requests", f"{prefix}:tokens" @@ -546,7 +567,7 @@ class RequestRateLimiterStash: parallel_slot: ParallelSlotAcquisition | None = None parallel_slot_release_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False, compare=False) reserved_tokens: int = 0 - reserved_model: str | None = None + reserved_model: RateLimitedModel | None = None reserved_scopes: frozenset[tuple[str, str]] = field(default_factory=frozenset) itpm_reserved_tokens: int = 0 itpm_reserved_scopes: frozenset[tuple[str, str]] = field(default_factory=frozenset) @@ -626,9 +647,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self, internal_usage_cache: InternalUsageCache, time_provider: Callable[[], datetime] | None = None, + model_group_resolver: Callable[[str], str | None] = _resolve_model_group_alias_via_proxy_router, ): self.internal_usage_cache = internal_usage_cache self._time_provider = time_provider or datetime.now + self._model_group_resolver = model_group_resolver if self.internal_usage_cache.dual_cache.redis_cache is not None: self.batch_rate_limiter_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script( BATCH_RATE_LIMITER_SCRIPT @@ -2346,6 +2369,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if response["overall_code"] == "OVER_LIMIT": self._handle_rate_limit_error(response, descriptors, requested_model) + def _rate_limited_model(self, requested_model: str | None) -> RateLimitedModel | None: + if not requested_model: + return None + return RateLimitedModel( + requested=requested_model, + group=self._model_group_resolver(requested_model) or requested_model, + ) + def create_organization_rate_limit_descriptor( self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None = None ) -> list[RateLimitDescriptor]: @@ -2367,43 +2398,28 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) ) - # Model specific org rate limits - if ( + model: Final = self._rate_limited_model(requested_model) + if model is None: + return descriptors + model_specific_tpm_limit: Final = model.limit_in( + get_model_rate_limit_from_metadata(user_api_key_dict, "organization_metadata", "model_tpm_limit") + ) + model_specific_rpm_limit: Final = model.limit_in( get_model_rate_limit_from_metadata(user_api_key_dict, "organization_metadata", "model_rpm_limit") - is not None - or get_model_rate_limit_from_metadata(user_api_key_dict, "organization_metadata", "model_tpm_limit") - is not None - ): - _tpm_limit_for_team_model: Final = ( - get_model_rate_limit_from_metadata(user_api_key_dict, "organization_metadata", "model_tpm_limit") or {} + ) + if model_specific_tpm_limit is None and model_specific_rpm_limit is None: + return descriptors + descriptors.append( + RateLimitDescriptor( + key="model_per_organization", + value=f"{user_api_key_dict.org_id}:{model.group}", + rate_limit={ + "requests_per_unit": model_specific_rpm_limit, + "tokens_per_unit": model_specific_tpm_limit, + "window_size": self.window_size, + }, ) - _rpm_limit_for_team_model: Final = ( - get_model_rate_limit_from_metadata(user_api_key_dict, "organization_metadata", "model_rpm_limit") or {} - ) - - should_check_rate_limit = False - if requested_model in _tpm_limit_for_team_model or requested_model in _rpm_limit_for_team_model: - should_check_rate_limit = True - - if should_check_rate_limit: - model_specific_tpm_limit = None - model_specific_rpm_limit = None - if requested_model in _tpm_limit_for_team_model: - model_specific_tpm_limit = _tpm_limit_for_team_model[requested_model] - if requested_model in _rpm_limit_for_team_model: - model_specific_rpm_limit = _rpm_limit_for_team_model[requested_model] - descriptors.append( - RateLimitDescriptor( - key="model_per_organization", - value=f"{user_api_key_dict.org_id}:{requested_model}", - rate_limit={ - "requests_per_unit": model_specific_rpm_limit, - "tokens_per_unit": model_specific_tpm_limit, - "window_size": self.window_size, - }, - ) - ) - + ) return descriptors def _add_model_per_key_rate_limit_descriptor( @@ -2425,34 +2441,22 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): get_key_model_tpm_limit, ) - if not requested_model: + model: Final = self._rate_limited_model(requested_model) + if model is None: return - - _tpm_limit_for_key_model = get_key_model_tpm_limit(user_api_key_dict, model_name=requested_model) - _rpm_limit_for_key_model = get_key_model_rpm_limit(user_api_key_dict, model_name=requested_model) - - if _tpm_limit_for_key_model is None and _rpm_limit_for_key_model is None: - return - - _tpm_limit_for_key_model = _tpm_limit_for_key_model or {} - _rpm_limit_for_key_model = _rpm_limit_for_key_model or {} - - # Check if model has any rate limits configured - should_check_rate_limit: Final = ( - requested_model in _tpm_limit_for_key_model or requested_model in _rpm_limit_for_key_model + model_specific_tpm_limit: Final = model.limit_in( + get_key_model_tpm_limit(user_api_key_dict, model_name=model.group) ) - - if not should_check_rate_limit: + model_specific_rpm_limit: Final = model.limit_in( + get_key_model_rpm_limit(user_api_key_dict, model_name=model.group) + ) + if model_specific_tpm_limit is None and model_specific_rpm_limit is None: return - # Get model-specific limits - model_specific_tpm_limit: Final[int | None] = _tpm_limit_for_key_model.get(requested_model) - model_specific_rpm_limit: Final[int | None] = _rpm_limit_for_key_model.get(requested_model) - descriptors.append( RateLimitDescriptor( key="model_per_key", - value=f"{user_api_key_dict.api_key}:{requested_model}", + value=f"{user_api_key_dict.api_key}:{model.group}", rate_limit={ "requests_per_unit": model_specific_rpm_limit, "tokens_per_unit": model_specific_tpm_limit, @@ -2955,32 +2959,30 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): def _key_owns_model_limit( self, user_api_key_dict: UserAPIKeyAuth, - requested_model: str, + model: RateLimitedModel, rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit"], ) -> bool: - key_own_limits: Final = get_key_own_model_rate_limit(user_api_key_dict, rate_limit_key) - return key_own_limits is not None and key_own_limits.get(requested_model) is not None + return model.limit_in(get_key_own_model_rate_limit(user_api_key_dict, rate_limit_key)) is not None def _inherited_team_model_limit( self, user_api_key_dict: UserAPIKeyAuth, - requested_model: str, + model: RateLimitedModel, rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit"], ) -> int | None: - team_limits: Final = get_model_rate_limit_from_metadata(user_api_key_dict, "team_metadata", rate_limit_key) - team_limit: Final = team_limits.get(requested_model) if team_limits else None - if team_limit is None: - return None - if self._key_owns_model_limit(user_api_key_dict, requested_model, rate_limit_key): + team_limit: Final = model.limit_in( + get_model_rate_limit_from_metadata(user_api_key_dict, "team_metadata", rate_limit_key) + ) + if team_limit is None or self._key_owns_model_limit(user_api_key_dict, model, rate_limit_key): return None return team_limit def _key_owns_model_tpm_limit_from_request_metadata( self, request_metadata: Mapping[str, object], - model_group: str | None, + model: RateLimitedModel | None, ) -> bool: - if model_group is None: + if model is None: return False key_view: Final = UserAPIKeyAuth.model_validate( { @@ -2988,7 +2990,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): "model_max_budget": request_metadata.get("user_api_key_model_max_budget") or {}, } ) - return self._key_owns_model_limit(key_view, model_group, "model_tpm_limit") + return self._key_owns_model_limit(key_view, model, "model_tpm_limit") def _add_team_model_rate_limit_descriptor_from_metadata( self, @@ -2996,16 +2998,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): requested_model: str | None, descriptors: list[RateLimitDescriptor], ) -> None: - if requested_model is None: + model: Final = self._rate_limited_model(requested_model) + if model is None: return - team_rpm_limit: Final = self._inherited_team_model_limit(user_api_key_dict, requested_model, "model_rpm_limit") - team_tpm_limit: Final = self._inherited_team_model_limit(user_api_key_dict, requested_model, "model_tpm_limit") + team_rpm_limit: Final = self._inherited_team_model_limit(user_api_key_dict, model, "model_rpm_limit") + team_tpm_limit: Final = self._inherited_team_model_limit(user_api_key_dict, model, "model_tpm_limit") if team_rpm_limit is None and team_tpm_limit is None: return descriptors.append( RateLimitDescriptor( key="model_per_team", - value=f"{user_api_key_dict.team_id}:{requested_model}", + value=f"{user_api_key_dict.team_id}:{model.group}", rate_limit={ "requests_per_unit": team_rpm_limit, "tokens_per_unit": team_tpm_limit, @@ -3021,34 +3024,28 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): descriptors: list[RateLimitDescriptor], ) -> None: """Add project model rate limit descriptor from project_metadata if applicable.""" - if ( - get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_rpm_limit") is not None - or get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_tpm_limit") is not None - ): - _tpm_limit_for_project_model: Final = ( - get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_tpm_limit") or {} + model: Final = self._rate_limited_model(requested_model) + if model is None: + return + model_specific_tpm_limit: Final = model.limit_in( + get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_tpm_limit") + ) + model_specific_rpm_limit: Final = model.limit_in( + get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_rpm_limit") + ) + if model_specific_tpm_limit is None and model_specific_rpm_limit is None: + return + descriptors.append( + RateLimitDescriptor( + key="model_per_project", + value=f"{user_api_key_dict.project_id}:{model.group}", + rate_limit={ + "requests_per_unit": model_specific_rpm_limit, + "tokens_per_unit": model_specific_tpm_limit, + "window_size": self.window_size, + }, ) - _rpm_limit_for_project_model: Final = ( - get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_rpm_limit") or {} - ) - should_check_rate_limit: Final = ( - requested_model in _tpm_limit_for_project_model or requested_model in _rpm_limit_for_project_model - ) - - if should_check_rate_limit and requested_model is not None: - model_specific_tpm_limit: Final = _tpm_limit_for_project_model.get(requested_model) - model_specific_rpm_limit: Final = _rpm_limit_for_project_model.get(requested_model) - descriptors.append( - RateLimitDescriptor( - key="model_per_project", - value=f"{user_api_key_dict.project_id}:{requested_model}", - rate_limit={ - "requests_per_unit": model_specific_rpm_limit, - "tokens_per_unit": model_specific_tpm_limit, - "window_size": self.window_size, - }, - ) - ) + ) def add_project_io_token_rate_limit_descriptors_from_metadata( self, @@ -3062,25 +3059,21 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): TPM descriptor above -- these give Bedrock Mantle-style separate input/output token quotas at the project level. """ - if requested_model is None or user_api_key_dict.project_id is None: + model: Final = self._rate_limited_model(requested_model) + if model is None or user_api_key_dict.project_id is None: return - itpm_limit_for_project_model: Final = ( + model_itpm_limit: Final = model.limit_in( get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_itpm_limit") - or {} # mutable-ok: metadata helper returns an optional mapping ) - otpm_limit_for_project_model: Final = ( + model_otpm_limit: Final = model.limit_in( get_model_rate_limit_from_metadata(user_api_key_dict, "project_metadata", "model_otpm_limit") - or {} # mutable-ok: metadata helper returns an optional mapping ) - model_itpm_limit: Final = itpm_limit_for_project_model.get(requested_model) - model_otpm_limit: Final = otpm_limit_for_project_model.get(requested_model) - if model_itpm_limit is None and model_otpm_limit is None: return - descriptor_value: Final = f"{user_api_key_dict.project_id}:{requested_model}" + descriptor_value: Final = f"{user_api_key_dict.project_id}:{model.group}" if model_itpm_limit is not None: descriptors.append( RateLimitDescriptor( @@ -3767,7 +3760,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # the (actual - reserved) delta to those — unreserved # scopes get charged the full actual usage instead. stash.reserved_tokens = estimated_tokens - stash.reserved_model = requested_model + stash.reserved_model = self._rate_limited_model(requested_model) stash.reserved_scopes = frozenset( (d["key"], d["value"]) for d in descriptors @@ -4475,9 +4468,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # 'metadata' and 'litellm_metadata' fields from litellm_params standard_logging_object: Final = kwargs.get("standard_logging_object") or {} request_metadata: Final = get_litellm_metadata_from_kwargs(kwargs) - if request_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY): - # Internal sub-calls bill spend to the caller but are not the caller's - # traffic; charging them here would let background evals eat TPM headroom. + origin: Final = request_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) + if origin and origin != "autorouter_compaction": + # Background evaluations keep their exemption; foreground compaction + # is necessary caller traffic and consumes the caller's token limits. return [] standard_logging_metadata: Final = standard_logging_object.get("metadata") or {} @@ -4513,9 +4507,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): reserved_scopes: Final[frozenset[tuple[str, str]]] = stash.reserved_scopes if stash is not None else frozenset() # Reconciliation must target the same model-scoped counter that the # pre-call reservation incremented. If a reservation was made, - # ``reserved_model`` is authoritative; otherwise fall back to the - # router's ``model_group`` (covers the no-reservation charge path). - reconcile_model: Final = reserved_model or model_group + # ``reserved_model`` (resolved at admission, so an alias map reload + # mid-flight cannot move the charge) is authoritative; otherwise fall + # back to the router's ``model_group`` (the no-reservation charge path). + reconcile_model: Final = reserved_model if reserved_model is not None else self._rate_limited_model(model_group) pipeline_operations: Final[list[RedisPipelineIncrementOperation]] = [] @@ -4533,7 +4528,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): targets: Final = self._collect_tpm_scope_targets( standard_logging_metadata=standard_logging_metadata, kwargs=kwargs, - model_group=reconcile_model, + model_group=reconcile_model.group if reconcile_model is not None else None, ) charged_targets: Final = ( [target for target in targets if target[0] != "model_per_team"] diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 08e8e4f8c10..81894a5ff12 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -11,6 +11,7 @@ from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, + budget_reservation_from_metadata, get_litellm_metadata_from_kwargs, ) from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup @@ -630,17 +631,7 @@ def _metadata_keys(metadata: object) -> tuple[str, ...]: def _get_budget_reservation_from_metadata(metadata: dict) -> dict | None: - metadata_budget_reservation: Final = metadata.get("user_api_key_budget_reservation") - if isinstance(metadata_budget_reservation, dict): - return metadata_budget_reservation - - user_api_key_auth_obj: Final = metadata.get("user_api_key_auth") - if user_api_key_auth_obj is None: - return None - if isinstance(user_api_key_auth_obj, dict): - budget_reservation: Final = user_api_key_auth_obj.get("budget_reservation") - return budget_reservation if isinstance(budget_reservation, dict) else None - return getattr(user_api_key_auth_obj, "budget_reservation", None) + return budget_reservation_from_metadata(metadata) def _get_request_tags_for_cost_tracking( diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 44d45dcd687..90bba82aa84 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2029,7 +2029,14 @@ async def add_litellm_data_to_request( _headers, allow_client_message_redaction_opt_out=_allow_client_message_redaction_opt_out, ) - _logging_safe_headers: Final = redact_credential_headers(_headers) + from litellm.proxy._experimental.mcp_server.utils import upstream_credential_headers + + _mcp_credential_headers: Final = upstream_credential_headers(_headers) + _logging_safe_headers: Final = redact_credential_headers( + MappingProxyType( + {name: value for name, value in _headers.items() if name.lower() not in _mcp_credential_headers} + ) + ) verbose_proxy_logger.debug("Request Headers: %s", _logging_safe_headers) verbose_proxy_logger.debug("Raw Headers: %s", _raw_headers) @@ -3116,7 +3123,15 @@ async def move_guardrails_to_metadata( - If guardrails not set on API key, then checks request metadata - Adds guardrails from policies attached to key/team metadata - Adds guardrails from policy engine based on team/key/model context + - Moves include_guardrail_response into request metadata before provider dispatch """ + if "include_guardrail_response" in data: + data[_metadata_variable_name][ + "include_guardrail_response" + ] = ( # rebind-ok: pre-call hooks mutate the shared request dict in place + data.pop("include_guardrail_response") is True + ) + # Early-out: skip all guardrails processing when nothing is configured key_metadata: Final = user_api_key_dict.metadata team_metadata: Final = user_api_key_dict.team_metadata diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 60ac7e55eaf..133181e9203 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -47,6 +47,8 @@ from litellm.proxy.common_utils.user_api_key_cache import ( object_permission_cache_key, user_object_permission_id_cache_key, ) +from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks from litellm.proxy.hooks.model_max_budget_limiter import build_model_max_budget_usage from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks from litellm.proxy.management_endpoints.common_daily_activity import ( @@ -2538,6 +2540,12 @@ async def delete_user( prisma_client=prisma_client, ) await _verification_token_table(prisma_client).delete_many(where=key_filter) + if keys_to_delete: + KeyManagementEventHooks.create_key_deleted_audit_logs( + keys_being_deleted=keys_to_delete, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) await delete_cache_key_objects( hashed_tokens=hashed_tokens_to_delete, user_api_key_cache=user_api_key_cache, @@ -2810,6 +2818,9 @@ async def ui_view_users( except HTTPException: raise except Exception as e: + if PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(e): + verbose_proxy_logger.warning("Database unavailable during user search: %s", type(e).__name__) + raise PrismaDBExceptionHandler.service_unavailable_proxy_exception(e) from e verbose_proxy_logger.exception("Error searching users: %s", e) raise HTTPException(status_code=500, detail=f"Error searching users: {e}") diff --git a/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py b/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py index 07234883062..292cec1346d 100644 --- a/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py +++ b/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py @@ -1,3 +1,4 @@ +import re from collections.abc import Mapping, Sequence from datetime import datetime from typing import Final, Protocol @@ -21,6 +22,51 @@ from litellm.repositories.table_repositories import JWTKeyMappingRepository router: Final = APIRouter() +_TOKEN_HASH_PATTERN: Final = re.compile(r"[0-9a-f]{64}") + + +def _validated_token_hash(token: str) -> str: + """Guards a plaintext key from being stored as a hash of a hash, which would never match.""" + if _TOKEN_HASH_PATTERN.fullmatch(token) is None: + raise HTTPException( + status_code=400, + detail=( + "`token` must be the SHA-256 hash of a virtual key " + "(64 lowercase hex characters). Pass the plaintext as `key` instead." + ), + ) + return token + + +_EXACTLY_ONE_IDENTIFIER: Final = ( + "Provide exactly one of `key` (the plaintext virtual key) or `token` (its SHA-256 hash)." +) +_AT_MOST_ONE_IDENTIFIER: Final = ( + "Provide at most one of `key` (the plaintext virtual key) or `token` (its SHA-256 hash)." +) + + +def _token_hash_for_create(data: CreateJWTKeyMappingRequest) -> str: + """Resolve the token hash to store, from either the plaintext key or its hash.""" + if data.key is not None and data.token is not None: + raise HTTPException(status_code=400, detail=_EXACTLY_ONE_IDENTIFIER) + if data.token is not None: + return _validated_token_hash(data.token) + if data.key is not None: + return hash_token(data.key) + raise HTTPException(status_code=400, detail=_EXACTLY_ONE_IDENTIFIER) + + +def _token_hash_for_update(data: UpdateJWTKeyMappingRequest) -> str | None: + """Resolve the token hash to store, or None to leave the mapped key alone.""" + if data.key is not None and data.token is not None: + raise HTTPException(status_code=400, detail=_AT_MOST_ONE_IDENTIFIER) + if data.token is not None: + return _validated_token_hash(data.token) + if data.key is not None: + return hash_token(data.key) + return None + class _JWTKeyMappingRecord(Protocol): """A ``LiteLLM_JWTKeyMapping`` row, viewed through the columns these endpoints read.""" @@ -111,7 +157,7 @@ async def create_jwt_key_mapping( raise HTTPException(status_code=500, detail="Database not connected") try: - hashed_key: Final = hash_token(data.key) + hashed_key: Final = _token_hash_for_create(data) create_data: Final = { "jwt_issuer": data.jwt_issuer or "", "jwt_claim_name": data.jwt_claim_name, @@ -166,9 +212,10 @@ async def update_jwt_key_mapping( if prisma_client is None: raise HTTPException(status_code=500, detail="Database not connected") - update_data: Final = data.model_dump(exclude_unset=True, exclude={"id", "key"}) - if data.key is not None: - update_data["token"] = hash_token(data.key) + update_data: Final = data.model_dump(exclude_unset=True, exclude={"id", "key", "token"}) + token_hash: Final = _token_hash_for_update(data) + if token_hash is not None: + update_data["token"] = token_hash if "jwt_issuer" in update_data: # DB column is NOT NULL (see schema.prisma); "" is the global/unscoped sentinel. update_data["jwt_issuer"] = update_data["jwt_issuer"] or "" diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index fbc7cf18003..1845f11c0fa 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -118,9 +118,6 @@ from litellm.proxy.management_helpers.team_member_permission_checks import ( from litellm.proxy.management_helpers.utils import management_endpoint_wrapper from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_start from litellm.proxy.spend_tracking.spend_tracking_utils import _is_master_key -from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( - get_ui_settings_cached, -) from litellm.proxy.utils import ( PrismaClient, ProxyLogging, @@ -487,8 +484,10 @@ async def _check_custom_key_allowed(custom_key_value: str | None) -> None: if custom_key_value is None: return - ui_settings: Final = await get_ui_settings_cached() - if ui_settings.get("disable_custom_api_keys", False) is True: + from litellm.proxy.config_resolvers.settings_rules import coerce_bool + from litellm.proxy.proxy_server import general_settings + + if coerce_bool(general_settings.get("disable_custom_api_keys", False)) is True: verbose_proxy_logger.warning("Custom API key rejected: disable_custom_api_keys is enabled") raise HTTPException( status_code=403, @@ -7591,25 +7590,47 @@ async def test_key_logging( _KEY_ALIAS_PATTERN: Final = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_\-/\.@]{0,253}[a-zA-Z0-9]$") +_KEY_ALIAS_PATTERN_MESSAGE: Final = ( + "Invalid key_alias format. Must be 2-255 characters, start/end with alphanumeric, and only contain a-zA-Z0-9_-/.@." +) +_KEY_ALIAS_MAX_LENGTH: Final = 255 + + +def parse_key_alias_pattern(value: object) -> str | None: + if value is None: + return None + if not isinstance(value, str): + raise ValueError( + f"Invalid regex set for litellm_settings.key_alias_pattern - value={value!r}: must be a string" + ) + try: + re.compile(value) + except re.error as e: + raise ValueError(f"Invalid regex set for litellm_settings.key_alias_pattern - value={value}: {e}") from e + return value + + +def _key_alias_rule() -> tuple[re.Pattern[str], str] | None: + if litellm.key_alias_pattern is not None: + return ( + re.compile(litellm.key_alias_pattern), + f"Invalid key_alias format. Must be at most {_KEY_ALIAS_MAX_LENGTH} characters and match the configured" + f" key_alias_pattern: {litellm.key_alias_pattern}", + ) + if litellm.enable_key_alias_format_validation: + return (_KEY_ALIAS_PATTERN, _KEY_ALIAS_PATTERN_MESSAGE) + return None def _validate_key_alias_format(key_alias: str | None) -> None: """ Validate the format of the key_alias. - A baseline validation always runs, regardless of - ``litellm.enable_key_alias_format_validation``. - - The remaining charset/length rules are gated behind - ``litellm.enable_key_alias_format_validation`` (default **False**). When disabled, - only the baseline validation above is performed, so existing workflows are not - broken. - - Rules (when enabled): - - None is OK (no alias). - - Otherwise must be 2–255 chars - - start/end with alphanumeric - - only allow a-zA-Z0-9_-/.@ + Path traversal and control characters are always rejected. The alias then has to + stay within ``_KEY_ALIAS_MAX_LENGTH`` and fully match ``litellm.key_alias_pattern`` + when one is configured, else the built-in pattern when + ``litellm.enable_key_alias_format_validation`` is on, else nothing more is checked + so existing workflows are not broken. """ if key_alias is None: return @@ -7624,12 +7645,14 @@ def _validate_key_alias_format(key_alias: str | None) -> None: code=400, ) - if not litellm.enable_key_alias_format_validation: + rule: Final = _key_alias_rule() + if rule is None: return - if not _KEY_ALIAS_PATTERN.match(key_alias): + pattern, message = rule + if len(key_alias) > _KEY_ALIAS_MAX_LENGTH or pattern.fullmatch(key_alias) is None: raise ProxyException( - message="Invalid key_alias format. Must be 2-255 characters, start/end with alphanumeric, and only contain a-zA-Z0-9_-/.@.", + message=message, type=ProxyErrorTypes.bad_request_error, param="key_alias", code=400, diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index ee0e4db291b..9ad78876043 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -983,7 +983,7 @@ if MCP_AVAILABLE: mcp_server_auth_headers=None, ) tools: Final = listing.tools - dumped_tools: Final = [dict(tool) for tool in tools] + dumped_tools: Final = [tool.model_dump(by_alias=True) for tool in tools] return {"tools": dumped_tools} diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 10a0a2f3104..fcadcfe2cae 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -14,12 +14,12 @@ import asyncio import datetime import json from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence -from contextlib import AbstractAsyncContextManager, asynccontextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager, suppress from dataclasses import dataclass from fnmatch import fnmatchcase from json import JSONDecodeError from types import MappingProxyType -from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeVar, cast, runtime_checkable +from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable from fastapi import APIRouter, Depends, Header, HTTPException, Request, status from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, field_validator @@ -29,6 +29,7 @@ from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import LITELLM_PROXY_ADMIN_NAME from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.ptu_pricing import ( CUSTOM_PRICING_FIELDS, PTU_EMPTIED_PRICING_FIELDS, @@ -139,7 +140,12 @@ from litellm.types.router import ( updateDeployment, updateLiteLLMParams, ) -from litellm.types.utils import echoed_cost_map_pricing_fields, without_server_derived_pricing +from litellm.types.utils import ( + COST_MAP_LOOKUP_KEY, + echoed_cost_map_fields, + echoed_cost_map_pricing_fields, + without_server_derived_pricing, +) from litellm.utils import get_utc_datetime if TYPE_CHECKING: @@ -928,7 +934,33 @@ def _ptu_priced_deployment(model_params: Deployment) -> Deployment: ) -def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> PrismaCompatibleUpdateDBModel: +def _cost_map_entry(db_model: Deployment, incoming_model_info: Mapping[str, object]) -> Mapping[str, object]: + base_model: Final = incoming_model_info.get("base_model") + lookup: Final = base_model if isinstance(base_model, str) else _decrypted_model(db_model.litellm_params.model) + if lookup is None: + return MappingProxyType({}) + with suppress(Exception): + return MappingProxyType(dict(litellm.get_model_info(model=lookup))) + return MappingProxyType({}) + + +LoadedCatalog: TypeAlias = Callable[[], Mapping[str, Mapping[str, object]]] # mutable-ok: Callable parameter syntax + + +def _loaded_catalog_entry( + incoming_model_info: Mapping[str, object], loaded_catalog: LoadedCatalog +) -> Mapping[str, object]: + catalog_key: Final = incoming_model_info.get(COST_MAP_LOOKUP_KEY) + if not isinstance(catalog_key, str): + return MappingProxyType({}) + return loaded_catalog().get(catalog_key, MappingProxyType({})) + + +def update_db_model( + db_model: Deployment, + updated_patch: updateDeployment, + loaded_catalog: LoadedCatalog = GetModelCostMap.loaded_model_cost_map, +) -> PrismaCompatibleUpdateDBModel: if updated_patch.model_info is not None: _raise_if_ptu_cost_attribution_disabled(updated_patch.model_info.model_dump(exclude_none=True)) merged_model_name: Final = updated_patch.model_name or db_model.model_name @@ -955,7 +987,24 @@ def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> Pr # update model info if updated_patch.model_info: - merged_model_info.update(without_server_derived_pricing(updated_patch.model_info.model_dump(exclude_none=True))) + incoming_model_info: Final = updated_patch.model_info.model_dump(exclude_none=True) + echoed_fields: Final = echoed_cost_map_fields( + incoming_model_info, + _cost_map_entry(db_model, incoming_model_info), + _loaded_catalog_entry(incoming_model_info, loaded_catalog), + ) + merged_model_info.update( + MappingProxyType( + dict( + (k, v) + for k, v in without_server_derived_pricing(incoming_model_info).items() + if k not in echoed_fields + ) + ) + ) + for k in echoed_fields: + if k in merged_model_info and merged_model_info[k] != incoming_model_info[k]: + del merged_model_info[k] # Honor explicit-null clears LAST, after both merges, so a model_info blob a client # passes through cannot silently undo a litellm_params clear via .update(). diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py index d6d74ada35a..a557e3a6082 100644 --- a/litellm/proxy/management_endpoints/router_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py @@ -33,6 +33,9 @@ class RouterSettingsResponse(BaseModel): fields: list[RouterSettingsField] = Field(description="List of all configurable router settings with metadata") current_values: dict[str, Any] = Field(description="Current values of router settings") routing_strategy_descriptions: dict[str, str] = Field(description="Descriptions for each routing strategy option") + routing_group_strategies: tuple[str, ...] = Field( + description="Strategies supported when constructing a routing group" + ) source: dict[str, FieldSource] = Field(description="Source of each current router setting") @@ -41,6 +44,9 @@ class RouterFieldsResponse(BaseModel): description="List of all configurable router settings with metadata (without field values)" ) routing_strategy_descriptions: dict[str, str] = Field(description="Descriptions for each routing strategy option") + routing_group_strategies: tuple[str, ...] = Field( + description="Strategies supported when constructing a routing group" + ) def _router_setting_source( @@ -114,7 +120,10 @@ async def get_router_settings( if llm_router is not None: # Router exposes routing groups as private `_routing_groups`; the # generic `hasattr` loop below would miss them. - current_values["routing_groups"] = [group.model_dump() for group in llm_router._routing_groups.values()] + current_values["routing_groups"] = [ + group.model_dump(exclude=frozenset(("model_priorities",)) if group.model_priorities is None else None) + for group in llm_router._routing_groups.values() + ] for field in router_fields: if field.field_name == "routing_groups": continue @@ -147,6 +156,7 @@ async def get_router_settings( fields=router_fields, current_values=current_values, routing_strategy_descriptions=ROUTING_STRATEGY_DESCRIPTIONS, + routing_group_strategies=(*available_routing_strategies, "priority"), source=source, ) except Exception as e: @@ -196,6 +206,7 @@ async def get_router_fields( return RouterFieldsResponse( fields=router_fields, routing_strategy_descriptions=ROUTING_STRATEGY_DESCRIPTIONS, + routing_group_strategies=(*available_routing_strategies, "priority"), ) except Exception as e: verbose_proxy_logger.error("Error fetching router fields: %s", e) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 0a142166bc5..d092fc2fbb7 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -118,6 +118,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_ from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars from litellm.proxy.common_utils.json_merge_patch import apply_json_merge_patch from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks from litellm.proxy.hooks.model_max_budget_limiter import ( build_model_max_budget_usage, resolve_model_budget, @@ -3751,6 +3752,13 @@ async def _team_member_delete( } ) + if keys_to_delete: + KeyManagementEventHooks.create_key_deleted_audit_logs( + keys_being_deleted=keys_to_delete, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + await delete_cache_team_object( team_id=data.team_id, team_alias=existing_team_row.team_alias, @@ -4469,6 +4477,13 @@ async def delete_team( await prisma_client.delete_data(team_id_list=data.team_ids, table_name="key") + if keys_to_delete: + KeyManagementEventHooks.create_key_deleted_audit_logs( + keys_being_deleted=keys_to_delete, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) + await _invalidate_deleted_key_cache( keys=keys_to_delete, user_api_key_cache=user_api_key_cache, diff --git a/litellm/proxy/management_helpers/bulk_user_deletion.py b/litellm/proxy/management_helpers/bulk_user_deletion.py index af51a194413..8b5b601fe8a 100644 --- a/litellm/proxy/management_helpers/bulk_user_deletion.py +++ b/litellm/proxy/management_helpers/bulk_user_deletion.py @@ -31,6 +31,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.auth_checks import delete_cache_key_objects, get_jwt_key_mapping_cache_keys_for_tokens from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem from litellm.proxy.management_endpoints.common_utils import ( @@ -93,20 +94,28 @@ class _TeamRemoval: team: LiteLLM_TeamTable removed: frozenset[str] matched: frozenset[int] - deleted_key_tokens: tuple[str, ...] + deleted_keys: tuple["prisma_models.LiteLLM_VerificationToken", ...] jwt_mapping_cache_keys: tuple[str, ...] + @property + def deleted_key_tokens(self) -> tuple[str, ...]: + return tuple(k.token for k in self.deleted_keys) + @dataclass(frozen=True, slots=True) class _UserBatchDeletion: removals: Mapping[str, _TeamRemoval] - deleted_key_tokens: tuple[str, ...] + deleted_keys: tuple["prisma_models.LiteLLM_VerificationToken", ...] jwt_mapping_cache_keys: tuple[str, ...] + @property + def deleted_key_tokens(self) -> tuple[str, ...]: + return tuple(k.token for k in self.deleted_keys) + @dataclass(frozen=True, slots=True) class _DeletedKeys: - tokens: tuple[str, ...] + keys: tuple["prisma_models.LiteLLM_VerificationToken", ...] jwt_mapping_cache_keys: tuple[str, ...] @@ -276,7 +285,7 @@ async def _remove_members_from_team( ), removed=cleanup_ids, matched=matched, - deleted_key_tokens=tuple(k.token for k in keys), + deleted_keys=tuple(keys), jwt_mapping_cache_keys=jwt_mapping_cache_keys, ) @@ -330,6 +339,12 @@ async def bulk_remove_team_members( members: Final = tuple(data.members[i] for i in kept_indexes) async with prisma_client.tx(timeout=_BATCH_TX_TIMEOUT) as tx: removal: Final = await _remove_members_from_team(prisma_client, tx, team_id, members, user_api_key_dict) + if removal.deleted_keys: + KeyManagementEventHooks.create_key_deleted_audit_logs( + keys_being_deleted=removal.deleted_keys, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) await delete_cache_key_objects( hashed_tokens=removal.deleted_key_tokens, user_api_key_cache=user_api_key_cache, @@ -407,7 +422,7 @@ async def _delete_user_rows( await _org_membership_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) await _membership_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) await _user_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) - return _DeletedKeys(tokens=tuple(k.token for k in keys), jwt_mapping_cache_keys=jwt_mapping_cache_keys) + return _DeletedKeys(keys=tuple(keys), jwt_mapping_cache_keys=jwt_mapping_cache_keys) async def _delete_users_tx( @@ -446,7 +461,7 @@ async def _delete_users_tx( ) return _UserBatchDeletion( removals=removals, - deleted_key_tokens=deleted_keys.tokens + tuple(t for r in removals.values() for t in r.deleted_key_tokens), + deleted_keys=deleted_keys.keys + tuple(k for r in removals.values() for k in r.deleted_keys), jwt_mapping_cache_keys=deleted_keys.jwt_mapping_cache_keys + tuple(k for r in removals.values() for k in r.jwt_mapping_cache_keys), ) @@ -469,6 +484,12 @@ async def _delete_users( except Exception as e: # noqa: BLE001 # the rolled-back batch is reported per row, not as a request failure verbose_proxy_logger.error("users/bulk_delete: failed to delete users %s: %s", sorted(user_ids), e) return _error_message(e) + if deletion.deleted_keys: + KeyManagementEventHooks.create_key_deleted_audit_logs( + keys_being_deleted=deletion.deleted_keys, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) await delete_cache_key_objects( hashed_tokens=deletion.deleted_key_tokens, user_api_key_cache=user_api_key_cache, @@ -555,7 +576,7 @@ async def bulk_delete_users( litellm_changed_by, ) if candidates - else _UserBatchDeletion(removals=MappingProxyType({}), deleted_key_tokens=(), jwt_mapping_cache_keys=()) + else _UserBatchDeletion(removals=MappingProxyType({}), deleted_keys=(), jwt_mapping_cache_keys=()) ) def result(index: int, user_id: str) -> UserDeleteResult: diff --git a/litellm/proxy/middleware/budget_reservation_release_middleware.py b/litellm/proxy/middleware/budget_reservation_release_middleware.py new file mode 100644 index 00000000000..f7ac885274e --- /dev/null +++ b/litellm/proxy/middleware/budget_reservation_release_middleware.py @@ -0,0 +1,33 @@ +from collections.abc import Awaitable, Callable, Mapping +from typing import Final + +from starlette.types import ASGIApp, Receive, Scope, Send + +_SCOPES_AUTH_STAMPS: Final = frozenset({"http", "websocket"}) + + +class BudgetReservationReleaseMiddleware: + """Releases the budget reservation auth made for a request once no callback owns it. + + Auth stamps the reservation on the request or socket state; a call that starts + claims it for the cost callbacks, which settle it on success or failure. When the + response has been sent or the socket has closed and the reservation is still + unclaimed, nothing else ever would, so it is released here instead of pinning the + spend counter until its TTL. + """ + + def __init__(self, app: ASGIApp, release: Callable[[Mapping[str, object]], Awaitable[None]]) -> None: + self.app = app + self.release = release + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] not in _SCOPES_AUTH_STAMPS: + await self.app(scope, receive, send) + return + try: + await self.app(scope, receive, send) + finally: + state: Final = scope.get("state") + budget_reservation: Final = state.get("budget_reservation") if isinstance(state, Mapping) else None + if isinstance(budget_reservation, Mapping): + await self.release(budget_reservation) diff --git a/litellm/proxy/native_compaction.py b/litellm/proxy/native_compaction.py new file mode 100644 index 00000000000..fd27e7fbbdc --- /dev/null +++ b/litellm/proxy/native_compaction.py @@ -0,0 +1,90 @@ +import asyncio +from collections.abc import Awaitable, Mapping +from contextvars import Context +from types import MappingProxyType +from typing import Final, Literal, TypeVar + +from fastapi import Request +from pydantic import TypeAdapter, ValidationError +from starlette.types import ASGIApp + +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + inherit_message_logging_privacy, + initialize_standard_callback_dynamic_params, +) +from litellm.llms.custom_httpx.asgi_handler import get_async_asgi_client +from litellm.proxy.litellm_pre_call_utils import UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS +from litellm.router_strategy.complexity_router.context_compaction import ( + compaction_executor, + native_compaction_call, +) + +_ResultT: Final = TypeVar("_ResultT") +_ASGI_APP: Final = TypeAdapter[ASGIApp](ASGIApp) +_ROOT_PATH: Final = TypeAdapter(str) +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) +_REMOVED_HEADERS: Final = frozenset( + ( + b"content-length", + b"x-litellm-call-id", + b"x-litellm-num-retries", + b"x-litellm-timeout", + b"x-litellm-stream-timeout", + ) +) + + +async def with_proxy_compaction_executor(call: Awaitable[_ResultT], request: Request) -> _ResultT: + async def execute( + protocol: Literal["chat", "messages"], payload: Mapping[str, object], parent_model: str | None = None + ) -> Mapping[str, object]: + logging_disabled: Final = initialize_standard_callback_dynamic_params().get("turn_off_message_logging") is True + + async def dispatch() -> Mapping[str, object]: + scope: Final = _JSON_OBJECT.validate_python(request.scope) + root_path: Final = _ROOT_PATH.validate_python(scope.get("root_path", "")) + path: Final = "/v1/chat/completions" if protocol == "chat" else "/v1/messages" + url: Final = str(request.url.replace(path=root_path.rstrip("/") + path, query="", fragment="")) + headers: Final = tuple( + (name, value) + for name, value in request.headers.raw + if name.lower() not in _REMOVED_HEADERS + and not (logging_disabled and name.decode("latin-1").lower() in UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS) + ) + with ( + native_compaction_call(parent_model, str(payload["model"])), + inherit_message_logging_privacy(logging_disabled), + ): + with get_async_asgi_client( + app=_ASGI_APP.validate_python(scope["app"]), + root_path=root_path, + client=request.client, + ) as client: + async with client.stream( + "POST", url, headers=headers, json=_JSON_OBJECT.validate_python(payload) + ) as response: + if not response.is_success: + raise BadRequestError( + message=f"Native compaction child request failed (HTTP {response.status_code})", + model="context_compaction", + llm_provider="", + ) + body: Final = await response.aread() + try: + return MappingProxyType(_JSON_OBJECT.validate_json(body)) + except ValidationError: + raise BadRequestError( + message="Native compaction child returned an invalid JSON object", + model="context_compaction", + llm_provider="", + ) from None + + task: Final = Context().run(asyncio.create_task, dispatch()) + return await task + + token: Final = compaction_executor.set(execute) + try: + return await call + finally: + compaction_executor.reset(token) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 44d9f11360d..2d071343844 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -14,6 +14,7 @@ import json import os import posixpath import re +import sys from collections.abc import AsyncGenerator, Callable, Mapping, Sequence from dataclasses import dataclass from functools import partial @@ -52,6 +53,7 @@ from litellm.llms.deepgram.common_utils import ( deepgram_listen_requested_model, deepgram_listen_websocket_target, ) +from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.passthrough.main import AsyncPassthroughStreamingResponse @@ -100,6 +102,12 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, ) +from litellm.types.passthrough_endpoints.tinyfish import ( + TINYFISH_AUTHENTICATED_RUN_FIELDS, + TINYFISH_PASSTHROUGH_TIMEOUT_SECONDS, + TINYFISH_REJECTED_ENVELOPE_FIELDS, + is_allowed_tinyfish_endpoint, +) from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials from litellm.types.router import LiteLLMParamsTypedDict from litellm.types.utils import LlmProviders @@ -421,6 +429,54 @@ async def cohere_proxy_route( return received_value +def _fal_target(endpoint: str) -> httpx.URL: + base_target_url: Final = fal_ai_queue_base() + encoded_endpoint: Final = httpx.URL(endpoint).path + normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}" + base_url: Final = httpx.URL(base_target_url) + return base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint), + ) + + +@router.api_route( + "/fal_ai/{endpoint:path}", + methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: FastAPI route metadata requires a list + tags=["Fal AI Pass-through", "pass-through"], # mutable-ok: FastAPI route metadata requires a list +) +async def fal_ai_proxy_route( + endpoint: str, + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +): + updated_url: Final = _fal_target(endpoint) + fal_ai_api_key: Final = passthrough_endpoint_router.get_credentials( + custom_llm_provider="fal_ai", + region_name=None, + ) + if fal_ai_api_key is None: + raise HTTPException( + status_code=401, + detail="FAL_AI_API_KEY is not set and no fal_ai pass-through deployment credentials are configured", + ) + if "/requests/" not in endpoint and fal_ai_passthrough_cost(endpoint, await _read_request_body(request)) is None: + raise HTTPException( + status_code=400, + detail=f"fal_ai/{endpoint} has no pricing entry for this request; only priced Fal requests can be submitted through /fal_ai", + ) + endpoint_func: Final = create_pass_through_route( + endpoint=endpoint, + target=str(updated_url), + custom_headers={ + "Authorization": f"Key {fal_ai_api_key}" + }, # mutable-ok: pass-through request headers require a mutable mapping + custom_llm_provider="fal_ai", + is_streaming_request=False, + ) + return await endpoint_func(request, fastapi_response, user_api_key_dict) + + @router.api_route( "/vllm/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], @@ -579,6 +635,42 @@ async def typesafe_proxy_route( return await endpoint_func(request, fastapi_response, user_api_key_dict) +@router.api_route( + "/openrouter/{endpoint:path}", + methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: FastAPI route metadata requires a list + tags=["OpenRouter Pass-through", "pass-through"], # mutable-ok: FastAPI route metadata requires a list +) +async def openrouter_proxy_route( + endpoint: str, + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +): + base_target_url: Final = get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" + api_root: Final = base_target_url.removesuffix("/").removesuffix("/v1") + encoded_endpoint: Final = httpx.URL(endpoint).path + normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}" + base_url: Final = httpx.URL(api_root) + updated_url: Final = base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint), + ) + openrouter_api_key: Final = passthrough_endpoint_router.get_credentials( + custom_llm_provider="openrouter", + region_name=None, + ) + endpoint_func: Final = create_pass_through_route( + endpoint=endpoint, + target=str(updated_url), + custom_headers={ # mutable-ok: pass-through request headers require a mutable mapping + "Authorization": f"Bearer {openrouter_api_key}", + "Content-Type": "application/json", + }, + custom_llm_provider="openrouter", + is_streaming_request=False, + ) + return await endpoint_func(request, fastapi_response, user_api_key_dict) + + @router.api_route( "/milvus/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], @@ -3270,6 +3362,138 @@ async def cursor_proxy_route( return received_value +TINYFISH_JSON_OBJECT_BODY_DETAIL: Final = ( + "TinyFish requests must be a JSON object body sent with Content-Type: application/json." +) + + +async def _tinyfish_json_object_field_names(request: Request) -> frozenset[str] | None: + content_type: Final = request.headers.get("content-type", "") + if content_type and not is_json_content_type(content_type): + return None + raw_body: Final = await request.body() + if not raw_body: + return frozenset() + try: + parsed: Final[object] = json.loads(raw_body) # any-ok: json.loads -> Any + except (json.JSONDecodeError, UnicodeDecodeError): + return None + return frozenset(parsed) if isinstance(parsed, dict) else None + + +def _tinyfish_route_timeout() -> float | None: + # only raise the 600s default to cover legal 1200s runs; an operator's configured timeout still wins + proxy_server: Final = sys.modules.get("litellm.proxy.proxy_server") + operator_settings: Final = getattr(proxy_server, "general_settings", None) + operator_timeout: Final = ( + operator_settings.get("pass_through_request_timeout") if isinstance(operator_settings, Mapping) else None + ) + return None if operator_timeout is not None else TINYFISH_PASSTHROUGH_TIMEOUT_SECONDS + + +@router.api_route( + "/tinyfish/{endpoint:path}", + methods=["GET", "POST"], # mutable-ok: fastapi api_route requires List[str] + tags=["TinyFish Pass-through", "pass-through"], # mutable-ok: fastapi api_route requires a list +) +async def tinyfish_proxy_route( + endpoint: str, + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), # noqa: B008 # FastAPI dependency injection +) -> Response: + """ + Pass-through for the TinyFish Agent API (goal-based web automation). + + Forwarded endpoints: + - POST /v1/automation/run — run to completion (blocking) + - POST /v1/automation/run-async — submit a run, poll GET /v1/runs/{id} for the result + - POST /v1/automation/run-sse — run with SSE progress events + - GET /v1/runs/{id} — run status / result + - POST /v1/runs/{id}/cancel — cancel a run + + Every other Agent API endpoint (vault, wallet, browser profiles, and the GET /v1/runs + listing, which would let any caller discover other callers' run ids) returns 403: all + proxy callers share one upstream key. + + Credential lookup order: + 1. passthrough_endpoint_router (config.yaml deployments with use_in_pass_through) + 2. TINYFISH_API_KEY environment variable + + [Docs](https://docs.litellm.ai/docs/pass_through/tinyfish) + """ + from .llm_provider_handlers.tinyfish_passthrough_logging_handler import ( + resolve_tinyfish_agent_api_base, + ) + + raw_endpoint_path: Final = httpx.URL(endpoint).path + encoded_endpoint: Final = raw_endpoint_path if raw_endpoint_path.startswith("/") else f"/{raw_endpoint_path}" + + if not is_allowed_tinyfish_endpoint(request.method, encoded_endpoint): + raise HTTPException( + status_code=403, + detail=f"{request.method} {encoded_endpoint} is not an allowed TinyFish Agent passthrough endpoint. " + "Allowed: POST /v1/automation/run, POST /v1/automation/run-async, POST /v1/automation/run-sse, " + "GET /v1/runs/{id}, POST /v1/runs/{id}/cancel.", + ) + + if request.method == "POST": + body_fields: Final = await _tinyfish_json_object_field_names(request) + if body_fields is None: + raise HTTPException(status_code=400, detail=TINYFISH_JSON_OBJECT_BODY_DETAIL) + envelope_fields: Final = tuple(sorted(body_fields & TINYFISH_REJECTED_ENVELOPE_FIELDS)) + if envelope_fields: + raise HTTPException( + status_code=400, + detail=f"Request fields [{', '.join(envelope_fields)}] are LiteLLM pass-through envelope controls " + "and are not accepted on the TinyFish route. Send the native TinyFish request body; streaming is " + "determined by the endpoint.", + ) + blocked_fields: Final = tuple(sorted(body_fields & TINYFISH_AUTHENTICATED_RUN_FIELDS)) + if ( + blocked_fields + and encoded_endpoint.startswith("/v1/automation/") + and str_to_bool(os.getenv("TINYFISH_ALLOW_AUTHENTICATED_RUNS")) is not True + ): + raise HTTPException( + status_code=403, + detail=f"Request fields [{', '.join(blocked_fields)}] run with the shared TinyFish account's saved " + "credentials and are disabled on this proxy. Ask the proxy admin to set " + "TINYFISH_ALLOW_AUTHENTICATED_RUNS=true to allow them.", + ) + + tinyfish_api_key: Final = passthrough_endpoint_router.get_credentials( + custom_llm_provider="tinyfish", + region_name=None, + ) + if tinyfish_api_key is None: + raise HTTPException( + status_code=401, + detail="TinyFish API key not found. Set the TINYFISH_API_KEY environment variable or add a " + "deployment with use_in_pass_through: true.", + ) + + base_url: Final = httpx.URL(resolve_tinyfish_agent_api_base()) + updated_url: Final = base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, encoded_endpoint) + ) + + endpoint_func: Final = create_pass_through_route( + endpoint=endpoint, + target=str(updated_url), + custom_headers=MappingProxyType({"X-API-Key": tinyfish_api_key}), + custom_llm_provider="tinyfish", + timeout=_tinyfish_route_timeout(), + ) + received_value: Final = await endpoint_func( + request, + fastapi_response, + user_api_key_dict, + ) + + return received_value + + VERTEX_LIVE_UNCONFIGURED_CLOSE_REASON: Final = ( "Vertex AI auth failed: set a use_in_pass_through vertex model, default_vertex_config, or DEFAULT_VERTEXAI_* env" ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/fal_ai_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/fal_ai_passthrough_logging_handler.py new file mode 100644 index 00000000000..3d1fad90e03 --- /dev/null +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/fal_ai_passthrough_logging_handler.py @@ -0,0 +1,71 @@ +from collections.abc import Mapping, Sequence +from typing import Final +from urllib.parse import urlparse + +import httpx + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base +from litellm.proxy._types import PassThroughEndpointLoggingTypedDict +from litellm.types.utils import ImageObject, ImageResponse + +FAL_AI_PROVIDER: Final[str] = litellm.LlmProviders.FAL_AI.value + + +def _url_parts(value: object) -> tuple[Mapping[str, object], ...]: + if isinstance(value, Mapping): + return (value,) if isinstance(value.get("url"), str) else () + if isinstance(value, Sequence) and not isinstance(value, str): + return tuple(item for item in value if isinstance(item, Mapping) and isinstance(item.get("url"), str)) + return () + + +class FalAIPassthroughLoggingHandler: + @staticmethod + def is_fal_ai_route(url_route: str, custom_llm_provider: str | None) -> bool: + return custom_llm_provider == FAL_AI_PROVIDER + + def fal_ai_passthrough_handler( + self, + response_body: Mapping[str, object], + request_body: Mapping[str, object], + logging_obj: LiteLLMLoggingObj, + url_route: str, + kwargs: Mapping[str, object], + ) -> PassThroughEndpointLoggingTypedDict: + base_path: Final = httpx.URL(fal_ai_queue_base()).path.strip("/") + raw_path: Final = urlparse(url_route).path.strip("/") + upstream_path: Final = raw_path.removeprefix(f"{base_path}/") if base_path else raw_path + model: Final = upstream_path.partition("/requests/")[0] + is_submit: Final = "/requests/" not in upstream_path + response: Final = ImageResponse( + data=tuple( + ImageObject(url=url) + for value in response_body.values() + for part in _url_parts(value) + if isinstance((url := part.get("url")), str) + ) + ) + response_cost: Final = fal_ai_passthrough_cost(model, request_body) if is_submit else None + response._hidden_params["response_cost"] = response_cost # pyright: ignore[reportPrivateUsage] # the logger reads a precomputed cost off the response's hidden params + logging_obj.model = model # rebind-ok: the spend logger reads model and cost off the shared logging object + logging_obj.model_call_details["model"] = model # rebind-ok: same shared logging object + logging_obj.model_call_details["custom_llm_provider"] = FAL_AI_PROVIDER # rebind-ok: same shared logging object + logging_obj.model_call_details["response_cost"] = response_cost # rebind-ok: same shared logging object + verbose_proxy_logger.debug( + "Fal AI passthrough cost tracking: model %s, cost %s", + model, + response_cost, + ) + logging_result: Final[PassThroughEndpointLoggingTypedDict] = { + "result": response, + "kwargs": { + **kwargs, + "model": model, + "custom_llm_provider": FAL_AI_PROVIDER, + "response_cost": response_cost, + }, + } + return logging_result diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py new file mode 100644 index 00000000000..a6c3cb669a6 --- /dev/null +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/tinyfish_passthrough_logging_handler.py @@ -0,0 +1,425 @@ +import asyncio +import json +import os +import time +import urllib.parse +from collections.abc import Mapping, Sequence +from datetime import datetime +from types import MappingProxyType +from typing import Final, NamedTuple +from urllib.parse import urlparse + +import httpx +from pydantic import TypeAdapter, ValidationError + +from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.litellm_logging import ( + get_standard_logging_object_payload, +) +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, +) +from litellm.proxy._types import PassThroughEndpointLoggingTypedDict +from litellm.types.llms.custom_http import httpxSpecialProvider +from litellm.types.passthrough_endpoints.tinyfish import ( + TINYFISH_AGENT_DEFAULT_API_BASE, + TINYFISH_DEFAULT_COST_PER_STEP, + TINYFISH_MAX_CONSECUTIVE_POLL_FAILURES, + TINYFISH_MAX_POLLING_SECONDS, + TINYFISH_MODEL_NAME, + TINYFISH_POLLING_INTERVAL_SECONDS, + TINYFISH_TERMINAL_RUN_STATUSES, + TinyfishRun, +) +from litellm.types.utils import StandardPassThroughResponseObject + +_RUN_ADAPTER: Final = TypeAdapter(TinyfishRun) + +_EMPTY_KWARGS: Final[Mapping[str, object]] = MappingProxyType({}) + + +class _TinyfishLoggingPayload(NamedTuple): + result: StandardPassThroughResponseObject + kwargs: Mapping[str, object] + + def as_handler_result(self) -> PassThroughEndpointLoggingTypedDict: + handler_result: Final[PassThroughEndpointLoggingTypedDict] = { + "result": self.result, + "kwargs": {**self.kwargs}, + } + return handler_result + + +# asyncio tasks are weakly referenced by the loop; hold them until done or they can vanish mid-poll +_BACKGROUND_BILLING_TASKS: Final[set["asyncio.Task[None]"]] = set() # mutable-ok: task registry + + +def _register_billing_task(task: "asyncio.Task[None]") -> None: + _BACKGROUND_BILLING_TASKS.add(task) + task.add_done_callback(_BACKGROUND_BILLING_TASKS.discard) + task.add_done_callback(_warn_if_cancelled) + + +def _warn_if_cancelled(task: "asyncio.Task[None]") -> None: + # CancelledError bypasses the poller's exception handler, so shutdown-time charge loss must be logged here + if task.cancelled(): + verbose_proxy_logger.warning("TinyFish passthrough: billing poller cancelled mid-poll; the run may go unbilled") + + +_SSE_POLLER_SPAWNED_KEY: Final = "tinyfish_sse_poller_spawned" + + +def mark_sse_poller_spawned(logging_obj: LiteLLMLoggingObj) -> None: + logging_obj.model_call_details[_SSE_POLLER_SPAWNED_KEY] = True # rebind-ok: request-scoped scratch dict + + +def sse_poller_spawned(logging_obj: LiteLLMLoggingObj) -> bool: + return logging_obj.model_call_details.get(_SSE_POLLER_SPAWNED_KEY) is True + + +def run_id_from_sse_frames(frames: bytes) -> str | None: + return _run_id_from_sse_chunks(frames.decode("utf-8", errors="replace").splitlines()) + + +def resolve_tinyfish_agent_api_base() -> str: + raw: Final = (os.getenv("TINYFISH_AGENT_API_BASE") or TINYFISH_AGENT_DEFAULT_API_BASE).rstrip("/") + # a schemeless override would silently break both routing and billing (urlparse hostname becomes None) + return raw if "://" in raw else f"https://{raw}" + + +def resolve_tinyfish_cost_per_step() -> float: + raw: Final = os.getenv("TINYFISH_COST_PER_STEP") + if raw is None: + return TINYFISH_DEFAULT_COST_PER_STEP + try: + return float(raw) + except ValueError: + verbose_proxy_logger.warning( + "TINYFISH_COST_PER_STEP=%r is not a number; using the default rate %s", + raw, + TINYFISH_DEFAULT_COST_PER_STEP, + ) + return TINYFISH_DEFAULT_COST_PER_STEP + + +def is_tinyfish_agent_url(url: str) -> bool: + hostname: Final = urlparse(url).hostname + return hostname is not None and hostname == urlparse(resolve_tinyfish_agent_api_base()).hostname + + +def _parse_run(payload: object) -> TinyfishRun | None: + try: + return _RUN_ADAPTER.validate_python(payload) + except ValidationError as e: + verbose_proxy_logger.warning("TinyFish passthrough: unexpected run object shape: %s", e) + return None + + +def _run_cost(run: TinyfishRun | None) -> float | None: + if run is None: + return None + # TinyFish only invoices COMPLETED runs, so FAILED/CANCELLED runs must charge the team $0 + if run.get("status") != "COMPLETED": + return None + num_of_steps: Final = run.get("num_of_steps") + if num_of_steps is None: + return None + return num_of_steps * resolve_tinyfish_cost_per_step() + + +class TinyFishPassthroughLoggingHandler: + @staticmethod + def should_log_request(request_method: str, url_route: str) -> bool: + """Only run submissions are billed; GET /v1/runs* polling and cancels never write spend rows.""" + return request_method == "POST" and "/v1/automation/" in urlparse(url_route).path + + @staticmethod + def is_run_async_route(url_route: str) -> bool: + return urlparse(url_route).path.endswith("/v1/automation/run-async") + + @staticmethod + def tinyfish_passthrough_handler( + httpx_response: httpx.Response, + response_body: Mapping[str, object] | None, + logging_obj: LiteLLMLoggingObj, + url_route: str, + result: str, + start_time: datetime, + end_time: datetime, + cache_hit: bool, + request_body: Mapping[str, object], + **kwargs: object, # kwargs-ok: the passthrough logging dispatch forwards shared logging kwargs to every handler + ) -> PassThroughEndpointLoggingTypedDict: + """Bill a blocking POST /v1/automation/run: the response is the terminal run object.""" + try: + run: Final = _parse_run(response_body) if response_body is not None else None + handler_payload: Final = TinyFishPassthroughLoggingHandler._build_logging_payload( + run=run, + logging_obj=logging_obj, + result=result, + start_time=start_time, + end_time=end_time, + kwargs=kwargs, + ).as_handler_result() + except Exception as e: # noqa: BLE001 # billing/logging must never break the relayed request + verbose_proxy_logger.exception("Error in TinyFish passthrough logging handler: %s", e) + fallback_payload: Final[PassThroughEndpointLoggingTypedDict] = { + "result": StandardPassThroughResponseObject(response=result), + "kwargs": kwargs, + } + return fallback_payload + return handler_payload + + @staticmethod + def start_async_run_billing( + response_body: Mapping[str, object] | None, + logging_obj: LiteLLMLoggingObj, + result: str, + start_time: datetime, + cache_hit: bool, + **kwargs: object, # kwargs-ok: shared logging kwargs, replayed into _handle_logging when the run finishes + ) -> None: + """Bill POST /v1/automation/run-async once, when the polled run turns terminal.""" + submitted: Final = _parse_run(response_body) if response_body is not None else None + run_id: Final = submitted.get("run_id") if submitted is not None else None + if not run_id: + verbose_proxy_logger.warning( + "TinyFish passthrough: run-async response carried no run_id; logging the request without cost" + ) + task: Final = asyncio.create_task( + TinyFishPassthroughLoggingHandler._poll_and_log( + run_id=run_id, + logging_obj=logging_obj, + result=result, + start_time=start_time, + cache_hit=cache_hit, + kwargs=kwargs, + ) + ) + _register_billing_task(task) + + @staticmethod + def start_sse_run_billing( + run_id: str, + litellm_logging_obj: LiteLLMLoggingObj, + start_time: datetime, + client: AsyncHTTPHandler | None = None, + ) -> None: + """Bill POST /v1/automation/run-sse once via a detached poller that outlives client disconnects.""" + mark_sse_poller_spawned(litellm_logging_obj) + task: Final = asyncio.create_task( + TinyFishPassthroughLoggingHandler._poll_and_log( + run_id=run_id, + logging_obj=litellm_logging_obj, + result="", + start_time=start_time, + cache_hit=litellm_logging_obj.model_call_details.get("cache_hit") is True, + kwargs=_EMPTY_KWARGS, + client=client, + ) + ) + _register_billing_task(task) + + @staticmethod + async def _poll_and_log( + run_id: str | None, + logging_obj: LiteLLMLoggingObj, + result: str, + start_time: datetime, + cache_hit: bool, + kwargs: Mapping[str, object], + client: AsyncHTTPHandler | None = None, + ) -> None: + from ..pass_through_endpoints import pass_through_endpoint_logging + + try: + run: Final = ( + await TinyFishPassthroughLoggingHandler._poll_until_terminal(run_id, client) if run_id else None + ) + run_end_time: Final = datetime.now() # noqa: DTZ005 # naive to match the start_time stamped by pass_through_request + payload: Final = TinyFishPassthroughLoggingHandler._build_logging_payload( + run=run, + logging_obj=logging_obj, + result=result, + start_time=start_time, + end_time=run_end_time, + kwargs=kwargs, + ) + await pass_through_endpoint_logging._handle_logging( # pyright: ignore[reportPrivateUsage] # shared passthrough logging dispatcher, same access as the assemblyai handler + logging_obj=logging_obj, + standard_logging_response_object=payload.result, + result=result, + start_time=start_time, + end_time=run_end_time, + cache_hit=cache_hit, + **payload.kwargs, + ) + except Exception as e: # noqa: BLE001 # billing/logging must never break the relayed request + verbose_proxy_logger.exception("[Non blocking logging error] TinyFish run-async billing failed: %s", e) + + @staticmethod + async def _poll_until_terminal( + run_id: str, + client: AsyncHTTPHandler | None = None, + poll_interval_seconds: float = TINYFISH_POLLING_INTERVAL_SECONDS, + ) -> TinyfishRun | None: + deadline: Final = time.monotonic() + TINYFISH_MAX_POLLING_SECONDS + last_run: TinyfishRun | None = None # rebind-ok: poll-loop state + consecutive_failures = 0 # rebind-ok: poll-loop state + while time.monotonic() < deadline: + run = await TinyFishPassthroughLoggingHandler._fetch_run(run_id, client) + if run is None: + # a single transient poll failure must not drop the run's charge + consecutive_failures += 1 + if consecutive_failures >= TINYFISH_MAX_CONSECUTIVE_POLL_FAILURES: + verbose_proxy_logger.warning( + "TinyFish passthrough: giving up on run %s after %s consecutive poll failures; " + "logging the request without cost", + run_id, + consecutive_failures, + ) + return last_run + else: + consecutive_failures = 0 + last_run = run + if (run.get("status") or "") in TINYFISH_TERMINAL_RUN_STATUSES: + return run + await asyncio.sleep(poll_interval_seconds) + verbose_proxy_logger.warning( + "TinyFish passthrough: run %s not terminal after %ss; logging the request without cost", + run_id, + TINYFISH_MAX_POLLING_SECONDS, + ) + return last_run + + @staticmethod + async def _fetch_run(run_id: str, client: AsyncHTTPHandler | None = None) -> TinyfishRun | None: + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + passthrough_endpoint_router, + ) + + api_key: Final = passthrough_endpoint_router.get_credentials(custom_llm_provider="tinyfish", region_name=None) + if api_key is None: + verbose_proxy_logger.warning("TinyFish passthrough: no API key available to poll run %s", run_id) + return None + if any(c in run_id for c in ("/", "\\", "#", "?")) or ".." in run_id: + verbose_proxy_logger.warning("TinyFish passthrough: invalid run_id %r", run_id) + return None + safe_run_id: Final = urllib.parse.quote(run_id, safe="") + resolved_client: Final = client or get_async_httpx_client( + llm_provider=httpxSpecialProvider.PassThroughEndpoint, + params={"timeout": 30.0}, # mutable-ok: get_async_httpx_client takes a plain dict of client params + ) + try: + # screenshots=none keeps the poll payload small (no per-step screenshot URLs needed) + response: Final = await resolved_client.get( + f"{resolve_tinyfish_agent_api_base()}/v1/runs/{safe_run_id}?screenshots=none", + headers={"X-API-Key": api_key}, # mutable-ok: httpx headers= takes a plain dict + ) + if not (200 <= response.status_code < 300): + verbose_proxy_logger.warning( + "TinyFish passthrough: GET /v1/runs/%s returned %s", safe_run_id, response.status_code + ) + return None + payload: Final[object] = response.json() # any-ok: httpx Response.json() -> Any + return _parse_run(payload) + except Exception as e: # noqa: BLE001 # billing/logging must never break the relayed request + verbose_proxy_logger.warning("[Non blocking logging error] TinyFish run fetch failed: %s", e) + return None + + @staticmethod + async def handle_logging_tinyfish_collected_chunks( + litellm_logging_obj: LiteLLMLoggingObj, + url_route: str, + start_time: datetime, + all_chunks: Sequence[str], + end_time: datetime, + client: AsyncHTTPHandler | None = None, + ) -> PassThroughEndpointLoggingTypedDict: + """Fallback for run-sse streams with no poller: logs the request, pricing via one GET if a run_id parses.""" + try: + run_id: Final = _run_id_from_sse_chunks(all_chunks) + if run_id is None: + verbose_proxy_logger.warning( + "TinyFish passthrough: no run_id in SSE stream; logging the request without cost" + ) + run: Final = await TinyFishPassthroughLoggingHandler._fetch_run(run_id, client) if run_id else None + payload: Final = TinyFishPassthroughLoggingHandler._build_logging_payload( + run=run, + logging_obj=litellm_logging_obj, + result="", + start_time=start_time, + end_time=end_time, + kwargs=_EMPTY_KWARGS, + ).as_handler_result() + except Exception as e: # noqa: BLE001 # billing/logging must never break the relayed request + verbose_proxy_logger.exception("Error in TinyFish SSE passthrough logging handler: %s", e) + fallback_payload: Final[PassThroughEndpointLoggingTypedDict] = { + "result": StandardPassThroughResponseObject(response=""), + "kwargs": {}, + } + return fallback_payload + return payload + + @staticmethod + def _build_logging_payload( + run: TinyfishRun | None, + logging_obj: LiteLLMLoggingObj, + result: str, + start_time: datetime, + end_time: datetime, + kwargs: Mapping[str, object], + ) -> _TinyfishLoggingPayload: + response_cost: Final = _run_cost(run) + updated_kwargs: Final = { # mutable-ok: the logging pipeline requires a plain kwargs dict + **kwargs, + "model": TINYFISH_MODEL_NAME, + "custom_llm_provider": "tinyfish", + "response_cost": response_cost, + # spend rows key on this as request_id; without it every poller-billed row is a NULL-key collision + "litellm_call_id": logging_obj.litellm_call_id, + # the poller paths pass no request kwargs, so SLO attribution (key hash, team, tags) needs the stored params + "litellm_params": kwargs.get("litellm_params") + or logging_obj.model_call_details.get("litellm_params") + or {}, # mutable-ok: the logging pipeline requires a plain kwargs dict + } + logging_obj.model_call_details.update( + model=TINYFISH_MODEL_NAME, + custom_llm_provider="tinyfish", + response_cost=response_cost, + ) + + logged_response: Final = StandardPassThroughResponseObject( + response=json.dumps(run) if run is not None else result + ) + standard_logging_object: Final = get_standard_logging_object_payload( + kwargs=updated_kwargs, + init_response_obj=logged_response, + start_time=start_time, + end_time=end_time, + logging_obj=logging_obj, + status="success", + ) + return _TinyfishLoggingPayload( + result=logged_response, + kwargs=MappingProxyType({**updated_kwargs, "standard_logging_object": standard_logging_object}), + ) + + +def _run_id_from_sse_chunks(all_chunks: Sequence[str]) -> str | None: + for line in all_chunks: + if not line.startswith("data:"): + continue + try: + event_payload: object = json.loads(line[5:].strip()) # any-ok: json.loads -> Any + except json.JSONDecodeError: + continue + event = _parse_run(event_payload) + if event is None: + continue + run_id = event.get("run_id") + if run_id: + return run_id + return None diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py index 9b196660c2c..887d17a7a20 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py @@ -65,6 +65,7 @@ class TypeSafePassthroughLoggingHandler: end_time: datetime, cache_hit: bool, request_body: Mapping[str, object], + custom_llm_provider: str, **kwargs: object, ) -> PassThroughEndpointLoggingTypedDict: response: Final = _parse_typesafe_response(response_body) @@ -72,12 +73,12 @@ class TypeSafePassthroughLoggingHandler: request_model_value: Final = request_body.get("model") request_model: Final = request_model_value if isinstance(request_model_value, str) else None logged_model: Final = response_model or request_model or "unknown" - model_name: Final = f"typesafe/{logged_model}" + model_name: Final = f"{custom_llm_provider}/{logged_model}" usage: Final = response.usage or _TypeSafeUsage() input_tokens: Final = usage.input_tokens output_tokens: Final = usage.output_tokens candidate_model_keys: Final = tuple( - f"typesafe/{model}" for model in (response_model, request_model) if model is not None + f"{custom_llm_provider}/{model}" for model in (response_model, request_model) if model is not None ) pricing: Final = _pricing_for(candidate_model_keys) response_cost: Final = ( @@ -91,13 +92,13 @@ class TypeSafePassthroughLoggingHandler: updated_kwargs: Final = { # mutable-ok: pass-through logging contract requires mutable kwargs **kwargs, "model": model_name, - "custom_llm_provider": "typesafe", + "custom_llm_provider": custom_llm_provider, "response_cost": response_cost, "combined_usage_object": usage_object, } logging_obj.model_call_details.update( model=model_name, - custom_llm_provider="typesafe", + custom_llm_provider=custom_llm_provider, response_cost=response_cost, ) standard_logging_object: Final = get_standard_logging_object_payload( diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 79a328f5199..c2874ac948f 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -47,6 +47,7 @@ from litellm.constants import ( from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( + bind_budget_reservation_to_callbacks, get_metadata_variable_name_from_kwargs, get_or_create_metadata_bucket, ) @@ -78,11 +79,13 @@ from litellm.proxy.common_request_processing import ( open_sse_before_first_byte, resolve_litellm_call_id, ) +from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id from litellm.proxy.common_utils.http_parsing_utils import ( _read_request_body, _safe_get_request_headers, ) from litellm.proxy.common_utils.openai_error_payload import ( + LITELLM_CALL_ID_HEADER, error_status_code, litellm_call_id_headers, openai_error_param, @@ -110,6 +113,9 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( ) from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, Usage +from .llm_provider_handlers.tinyfish_passthrough_logging_handler import ( + is_tinyfish_agent_url, +) from .streaming_handler import PassThroughStreamingHandler from .success_handler import PassThroughEndpointLogging from .upstream_usage_headers import ( @@ -380,6 +386,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): or (parsed_url.hostname and "openai.com" in parsed_url.hostname) ): return EndpointType.OPENAI + elif is_tinyfish_agent_url(url): + return EndpointType.TINYFISH return EndpointType.GENERIC @staticmethod @@ -1127,6 +1135,9 @@ async def pass_through_request( from litellm.proxy.proxy_server import ( general_settings as proxy_general_settings, ) + from litellm.proxy.proxy_server import ( + general_settings_view, + ) _managed_id_provider: Final = resolve_passthrough_managed_id_provider(custom_llm_provider) @@ -1632,6 +1643,7 @@ async def pass_through_request( **kwargs, ) ) + bind_budget_reservation_to_callbacks(logging_obj.litellm_params) ## CUSTOM HEADERS - `x-litellm-*` custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( @@ -1656,11 +1668,24 @@ async def pass_through_request( headers=response.headers, custom_headers=custom_headers, ) + emitted_call_id: Final = ( + JSON_OBJECT.validate_python(response_headers).get(LITELLM_CALL_ID_HEADER) + if response.status_code >= 400 + else None + ) + error_call_id: Final = ( + error_body_call_id(general_settings_view(), emitted_call_id) if isinstance(emitted_call_id, str) else None + ) + relayed_content: Final = ( + json.dumps(with_call_id(JSON_OBJECT.validate_python(response_body), error_call_id)).encode("utf-8") + if error_call_id is not None and isinstance(response_body, dict) + else content + ) if _content_modified: response_headers.pop("content-length", None) return Response( - content=content, + content=relayed_content, status_code=response.status_code, headers=response_headers, ) @@ -2543,6 +2568,7 @@ async def websocket_passthrough_request( **success_kwargs, ) ) + bind_budget_reservation_to_callbacks(logging_obj.litellm_params) # Call the proxy logging success hook if proxy_logging_obj: @@ -2714,6 +2740,7 @@ async def _relay_passthrough_response_bytes( **success_handler_kwargs, ) ) + bind_budget_reservation_to_callbacks(logging_obj.litellm_params) def _extract_model_from_vertex_ai_setup(setup_response: Mapping[str, object]) -> str | None: diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index fe9e104789b..e1f13f2bee0 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -9,6 +9,7 @@ import httpx import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.core_helpers import bind_budget_reservation_to_callbacks from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.proxy._types import PassThroughEndpointLoggingResultValues @@ -26,6 +27,11 @@ from .llm_provider_handlers.gemini_passthrough_logging_handler import ( from .llm_provider_handlers.openai_passthrough_logging_handler import ( OpenAIPassthroughLoggingHandler, ) +from .llm_provider_handlers.tinyfish_passthrough_logging_handler import ( + TinyFishPassthroughLoggingHandler, + run_id_from_sse_frames, + sse_poller_spawned, +) from .llm_provider_handlers.vertex_passthrough_logging_handler import ( VertexPassthroughLoggingHandler, ) @@ -69,6 +75,9 @@ class PassThroughStreamingHandler: exception: Exception, stream_context: PassThroughStreamContext | None = None, ) -> None: + # the tinyfish poller writes the one authoritative row; a failure row here would collide on its request_id + if endpoint_type == EndpointType.TINYFISH and sse_poller_spawned(litellm_logging_obj): + return await asyncify(PassThroughStreamingHandler._record_partial_usage_for_failure)( litellm_logging_obj=litellm_logging_obj, endpoint_type=endpoint_type, @@ -178,12 +187,25 @@ class PassThroughStreamingHandler: ) ) ) + # TinyFish SSE bills via a detached poller spawned on the first run_id frame, so disconnects can't lose the charge + tinyfish_scan_active = endpoint_type == EndpointType.TINYFISH # rebind-ok: scan stops once the poller spawns + tinyfish_pending = b"" # rebind-ok: SSE frame reassembly buffer across transport chunks try: if not cost_injection_active: # Hot path: just buffer for end-of-stream logging and forward. async for chunk in response.aiter_bytes(): raw_bytes.append(chunk) PassThroughStreamingHandler._stamp_first_chunk_if_needed(litellm_logging_obj) + if tinyfish_scan_active: + complete_frames, tinyfish_pending = split_complete_sse_frames(tinyfish_pending + chunk) + run_id = run_id_from_sse_frames(complete_frames) if b"run_id" in complete_frames else None + if run_id: + TinyFishPassthroughLoggingHandler.start_sse_run_billing( + run_id=run_id, + litellm_logging_obj=litellm_logging_obj, + start_time=start_time, + ) + tinyfish_scan_active = False yield chunk else: # ``cost_injection_active`` already requires ``model_name`` to @@ -218,6 +240,7 @@ class PassThroughStreamingHandler: and response.status_code < 400 ): logging_scheduled = True + bind_budget_reservation_to_callbacks(litellm_logging_obj.litellm_params) litellm_logging_obj._deferred_stream_complete_args = (_build_logging_coroutine(),) except Exception as e: verbose_proxy_logger.error("Error in chunk_processor: %s", e) @@ -250,6 +273,8 @@ class PassThroughStreamingHandler: GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=_build_logging_coroutine()) except Exception as e: verbose_proxy_logger.error("Error scheduling chunk_processor logging: %s", e) + else: + bind_budget_reservation_to_callbacks(litellm_logging_obj.litellm_params) @staticmethod async def _route_streaming_logging_to_handler( @@ -290,6 +315,37 @@ class PassThroughStreamingHandler: and not _is_provider_error_chunk(complete_frames) ) try: + # TinyFish billing is owned by the detached poller; the $0 fallback below is only for streams with no run_id + if endpoint_type == EndpointType.TINYFISH: + if sse_poller_spawned(litellm_logging_obj): + return + late_run_id: Final = run_id_from_sse_frames(b"".join(raw_bytes)) + if late_run_id: + # the run_id arrived in an unterminated frame; poll to terminal instead of mispricing a RUNNING run + TinyFishPassthroughLoggingHandler.start_sse_run_billing( + run_id=late_run_id, + litellm_logging_obj=litellm_logging_obj, + start_time=start_time, + ) + return + tinyfish_payload: Final = ( + await TinyFishPassthroughLoggingHandler.handle_logging_tinyfish_collected_chunks( + litellm_logging_obj=litellm_logging_obj, + url_route=url_route, + start_time=start_time, + all_chunks=PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(raw_bytes), + end_time=end_time, + ) + ) + await litellm_logging_obj.dispatch_success_handlers( + result=tinyfish_payload["result"], + start_time=start_time, + end_time=end_time, + cache_hit=litellm_logging_obj.model_call_details.get("cache_hit") is True, + prefer_async_handlers=True, + **tinyfish_payload["kwargs"], + ) + return ( standard_logging_response_object, kwargs, diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index de1a8ae1d93..6bba879b6c1 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -29,9 +29,16 @@ from .llm_provider_handlers.cursor_passthrough_logging_handler import ( from .llm_provider_handlers.deepgram_listen_passthrough_logging_handler import ( DeepgramListenPassthroughLoggingHandler, ) +from .llm_provider_handlers.fal_ai_passthrough_logging_handler import ( + FalAIPassthroughLoggingHandler, +) from .llm_provider_handlers.gemini_passthrough_logging_handler import ( GeminiPassthroughLoggingHandler, ) +from .llm_provider_handlers.tinyfish_passthrough_logging_handler import ( + TinyFishPassthroughLoggingHandler, + is_tinyfish_agent_url, +) from .llm_provider_handlers.transcribe_passthrough_logging_handler import ( TRANSCRIBE_CUSTOM_LLM_PROVIDER, PassThroughLogDispatch, @@ -278,6 +285,22 @@ class PassThroughEndpointLogging: ) standard_logging_response_object = comprehend_medical_handler_result["result"] # rebind-ok: elif-chain kwargs = comprehend_medical_handler_result["kwargs"] # rebind-ok: elif-chain contract + elif self.is_tinyfish_route(url_route, custom_llm_provider): + tinyfish_handler_result: Final = TinyFishPassthroughLoggingHandler.tinyfish_passthrough_handler( + httpx_response=httpx_response, + response_body=response_body if isinstance(response_body, dict) else None, + logging_obj=logging_obj, + url_route=url_route, + result=result, + start_time=start_time, + end_time=end_time, + cache_hit=cache_hit, + request_body=request_body, + **kwargs, + ) + standard_logging_response_object = tinyfish_handler_result["result"] # rebind-ok: elif-chain + kwargs = tinyfish_handler_result["kwargs"] # rebind-ok: elif-chain contract + elif self.is_azure_speech_route(custom_llm_provider): from .llm_provider_handlers.azure_speech_passthrough_logging_handler import ( AzureSpeechPassthroughLoggingHandler, @@ -311,7 +334,9 @@ class PassThroughEndpointLogging: ) standard_logging_response_object = transcribe_handler_result["result"] # rebind-ok: elif-chain kwargs = transcribe_handler_result["kwargs"] # rebind-ok: elif-chain contract - elif self.is_typesafe_route(custom_llm_provider): + elif self.is_typesafe_route(custom_llm_provider) or self.is_openrouter_decisions_route( + url_route, custom_llm_provider + ): from .llm_provider_handlers.typesafe_passthrough_logging_handler import ( TypeSafePassthroughLoggingHandler, ) @@ -326,10 +351,12 @@ class PassThroughEndpointLogging: end_time=end_time, cache_hit=cache_hit, request_body=request_body, + custom_llm_provider=custom_llm_provider or "", **kwargs, ) standard_logging_response_object = typesafe_handler_result["result"] kwargs = typesafe_handler_result["kwargs"] + elif self.is_vertex_ai_live_route(url_route): from .llm_provider_handlers.vertex_ai_live_passthrough_logging_handler import ( VertexAILivePassthroughLoggingHandler, @@ -367,6 +394,16 @@ class PassThroughEndpointLogging: ) standard_logging_response_object = deepgram_handler_result["result"] # rebind-ok: elif-chain kwargs = deepgram_handler_result["kwargs"] # rebind-ok: elif-chain contract + elif FalAIPassthroughLoggingHandler.is_fal_ai_route(url_route, custom_llm_provider): + fal_ai_handler_result: Final = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body=response_body if isinstance(response_body, dict) else MappingProxyType({}), + request_body=request_body, + logging_obj=logging_obj, + url_route=url_route, + kwargs=kwargs, + ) + standard_logging_response_object = fal_ai_handler_result["result"] # rebind-ok: elif-chain + kwargs = fal_ai_handler_result["kwargs"] # rebind-ok: elif-chain contract return_dict["standard_logging_response_object"] = standard_logging_response_object return_dict["kwargs"] = kwargs @@ -389,6 +426,20 @@ class PassThroughEndpointLogging: ): standard_logging_response_object: PassThroughEndpointLoggingResultValues | None = None logging_obj.model_call_details["passthrough_logging_payload"] = passthrough_logging_payload + if self.is_tinyfish_route(url_route, custom_llm_provider): + # polls and cancels never write spend rows; run-async bills once from the background poller + if not TinyFishPassthroughLoggingHandler.should_log_request(httpx_response.request.method, url_route): + return + if TinyFishPassthroughLoggingHandler.is_run_async_route(url_route): + TinyFishPassthroughLoggingHandler.start_async_run_billing( + response_body=response_body if isinstance(response_body, dict) else None, + logging_obj=logging_obj, + result=result, + start_time=start_time, + cache_hit=cache_hit, + **kwargs, + ) + return if self.is_assemblyai_route(url_route) and not self.is_azure_speech_route(custom_llm_provider): if AssemblyAIPassthroughLoggingHandler._should_log_request(httpx_response.request.method) is not True: return @@ -496,6 +547,9 @@ class PassThroughEndpointLogging: def is_comprehend_medical_route(self, custom_llm_provider: str | None) -> bool: return custom_llm_provider == "comprehendmedical" + def is_tinyfish_route(self, url_route: str, custom_llm_provider: str | None) -> bool: + return custom_llm_provider == "tinyfish" or is_tinyfish_agent_url(url_route) + def is_azure_speech_route(self, custom_llm_provider: str | None) -> bool: return custom_llm_provider == AZURE_SPEECH_CUSTOM_LLM_PROVIDER @@ -505,6 +559,9 @@ class PassThroughEndpointLogging: def is_typesafe_route(self, custom_llm_provider: str | None) -> bool: return custom_llm_provider == "typesafe" + def is_openrouter_decisions_route(self, url_route: str, custom_llm_provider: str | None) -> bool: + return custom_llm_provider == "openrouter" and urlparse(url_route).path.endswith("/alpha/decisions") + def is_langfuse_route(self, url_route: str): parsed_url: Final = urlparse(url_route) for route in self.TRACKED_LANGFUSE_ROUTES: diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 78885461724..ac69f2e3894 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -33,21 +33,20 @@ else: FastAPI = Any -def _deprioritize_script_dir_in_sys_path() -> None: +def _drop_script_dir_from_sys_path() -> None: """Stop ``litellm/proxy`` modules from shadowing installed packages. Running this file as a script puts its own directory at ``sys.path[0]``, so ``import a2a`` resolves to ``litellm/proxy/a2a`` instead of the ``a2a`` SDK - and A2A agent calls fail. The entry is moved to the end rather than dropped, - because the sibling-import fallbacks in this module (``from proxy_server - import ...``) still need it. No-op under the ``litellm`` console script. + and ``proxy_server`` resolves to a second copy of + ``litellm.proxy.proxy_server``. No-op under the ``litellm`` console script. """ script_dir: Final = os.path.dirname(os.path.abspath(__file__)) if sys.path and os.path.abspath(sys.path[0]) == script_dir: - sys.path.append(sys.path.pop(0)) + sys.path.pop(0) -_deprioritize_script_dir_in_sys_path() +_drop_script_dir_from_sys_path() sys.path.append(os.getcwd()) config_filename: Final = "litellm.secrets" @@ -881,7 +880,7 @@ class ProxyInitializationHelpers: default=False, help="Use prisma db push instead of prisma migrate for database schema updates", ) -@click.option("--local", is_flag=True, default=False, help="for local debugging") +@click.option("--local", is_flag=True, default=False, help="no-op, kept for backwards compatibility") @click.option( "--skip_server_startup", is_flag=True, @@ -1058,35 +1057,15 @@ def run_server( return args: Final = locals() - if local: - from proxy_server import ( + try: + from litellm.proxy.proxy_server import ( KeyManagementSettings, ProxyConfig, app, save_worker_config, ) - else: - try: - from .proxy_server import ( - KeyManagementSettings, - ProxyConfig, - app, - save_worker_config, - ) - except ModuleNotFoundError as e: - raise ModuleNotFoundError(f"Missing dependency {e}. Run `pip install 'litellm[proxy]'`") - except ImportError as e: - if "litellm[proxy]" in str(e): - # user is missing a proxy dependency, ask them to pip install litellm[proxy] - raise e - else: - # this is just a local/relative import error, user git cloned litellm - from proxy_server import ( - KeyManagementSettings, - ProxyConfig, - app, - save_worker_config, - ) + except ModuleNotFoundError as e: + raise ModuleNotFoundError(f"Missing dependency {e}. Run `pip install 'litellm[proxy]'`") from e if version is True: ProxyInitializationHelpers._echo_litellm_version() return @@ -1283,6 +1262,7 @@ def run_server( if os.getenv("DATABASE_URL", None) is not None or os.getenv("DIRECT_URL", None) is not None: from litellm.proxy.db.db_url_settings import ( + DISABLE_PREPARED_STATEMENTS_ENV_VAR, add_missing_query_params, idle_lifetime_params, reader_shareable_params, @@ -1305,12 +1285,16 @@ def run_server( sys.exit(1) from litellm.secret_managers.main import get_secret + env_disable_prepared_statements: Final = token_auth_flag_enabled( + os.getenv(DISABLE_PREPARED_STATEMENTS_ENV_VAR), env_var=DISABLE_PREPARED_STATEMENTS_ENV_VAR + ) + disable_prepared_statements: Final = db_disable_prepared_statements or env_disable_prepared_statements connection_url_params: Final = _build_db_connection_url_params( connection_limit=db_connection_pool_limit, pool_timeout=db_connection_timeout, connect_timeout=db_connect_timeout, socket_timeout=db_socket_timeout, - disable_prepared_statements=db_disable_prepared_statements, + disable_prepared_statements=disable_prepared_statements, extra_params=db_extra_connection_params, ) lifetime_params: Final = idle_lifetime_params(general_settings.get("database_max_idle_connection_lifetime")) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 534dcf418cc..acb36fc4a59 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -75,6 +75,11 @@ from litellm.constants import ( RUNTIME_UPDATABLE_ROUTER_SETTINGS, ) from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.bug_report import ( + allowlisted, + bug_report_notice, + should_report_bug, +) from litellm.litellm_core_utils.litellm_logging import ( _init_custom_logger_compatible_class, ) @@ -127,6 +132,7 @@ from litellm.proxy.common_utils.callback_utils import ( strip_callback_config, ) from litellm.proxy.common_utils.realtime_utils import _realtime_request_body +from litellm.router_utils.access_windows import access_windows_config_error from litellm.router_utils.add_retry_fallback_headers import ( get_fallback_errors_from_headers, get_hidden_params_dict, @@ -304,6 +310,7 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.realtime_errors import ( + close_after_upstream_handshake_refusal, realtime_error_event, websocket_close_reason, ) @@ -367,10 +374,12 @@ from litellm.proxy.auth.user_api_key_auth import ( user_api_key_auth_websocket, ) from litellm.proxy.batches_endpoints.endpoints import router as batches_router +from litellm.proxy.bug_report_config import build_proxy_bug_report ## Import All Misc routes here ## from litellm.proxy.caching_routes import router as caching_router from litellm.proxy.common_request_processing import ( + KNOWN_PROXY_ROUTES, ProxyBaseLLMRequestProcessing, _is_azure_model_router_request, _should_return_raw_model_name, @@ -393,6 +402,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) +from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -418,6 +428,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( remove_sensitive_info_from_deployment, ) from litellm.proxy.common_utils.openai_error_payload import ( + LITELLM_CALL_ID_HEADER, headers_with_litellm_call_id, litellm_call_id_headers, with_litellm_call_id, @@ -585,6 +596,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( delete_verification_tokens, duration_in_seconds, generate_key_helper_fn, + parse_key_alias_pattern, ) from litellm.proxy.management_endpoints.key_management_endpoints import ( router as key_management_router, @@ -652,6 +664,9 @@ from litellm.proxy.middleware.billable_request_metrics_middleware import ( BillableRequestMetricsMiddleware, BillingRecorder, ) +from litellm.proxy.middleware.budget_reservation_release_middleware import ( + BudgetReservationReleaseMiddleware, +) from litellm.proxy.plugin_routes import ( register_plugins_from_config, ) @@ -727,7 +742,10 @@ from litellm.proxy.shutdown.scheduled_jobs import ( pause_scheduled_jobs, stop_in_flight_scheduler_jobs, ) -from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_start +from litellm.proxy.spend_tracking.budget_reservation import ( + get_budget_window_start, + release_unbound_budget_reservation, +) from litellm.proxy.spend_tracking.daily_global_spend_rollup import ( run_scheduled_daily_global_spend_reconcile, ) @@ -1479,23 +1497,29 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: ## Initialize shared aiohttp session for connection reuse shared_aiohttp_session = await _initialize_shared_aiohttp_session() - model_info_scheduler: Final = scheduler if scheduler is not None else AsyncIOScheduler() - model_info_scheduler.add_job( - ProxyStartupEvent.refresh_model_info, - "interval", - seconds=MODEL_INFO_REFRESH_SECONDS, - id="refresh_model_info", - next_run_time=datetime.now(timezone.utc), - max_instances=1, - replace_existing=True, + model_info_refresh_disabled: Final = ( + "disable_model_info_refresh" in general_settings and general_settings["disable_model_info_refresh"] is True ) - if not model_info_scheduler.running: - model_info_scheduler.start() + model_info_scheduler: Final = ( + None if model_info_refresh_disabled else scheduler if scheduler is not None else AsyncIOScheduler() + ) + if model_info_scheduler is not None: + model_info_scheduler.add_job( + ProxyStartupEvent.refresh_model_info, + "interval", + seconds=MODEL_INFO_REFRESH_SECONDS, + id="refresh_model_info", + next_run_time=datetime.now(timezone.utc), + max_instances=1, + replace_existing=True, + ) + if not model_info_scheduler.running: + model_info_scheduler.start() # End of startup event yield - if model_info_scheduler.running: + if model_info_scheduler is not None and model_info_scheduler.running: model_info_scheduler.remove_job("refresh_model_info") if model_info_scheduler is not scheduler: model_info_scheduler.shutdown(wait=False) @@ -1813,7 +1837,10 @@ async def openai_exception_handler(request: Request, exc: ProxyException): # NOTE: DO NOT MODIFY THIS, its crucial to map to Openai exceptions _log_model_access_denial(exc) headers: Final = exc.headers - error_dict: Final = exc.to_dict() + error_dict: Final = with_call_id( + JSON_OBJECT.validate_python(exc.to_dict()), + error_body_call_id(general_settings_view(), headers.get(LITELLM_CALL_ID_HEADER)), + ) status_code: Final = int(exc.code) if exc.code else status.HTTP_500_INTERNAL_SERVER_ERROR _close_dangling_otel_server_span(request, status_code, exc=exc) return JSONResponse( @@ -1953,7 +1980,21 @@ async def otel_request_validation_exception_handler(request: Request, exc: Reque async def otel_unhandled_exception_handler(request: Request, exc: Exception): if isinstance(exc, (ProxyException, HTTPException, RequestValidationError)): raise exc + if PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(exc): + verbose_proxy_logger.warning("Database unavailable during request: %s", type(exc).__name__) + return await openai_exception_handler( + request=request, exc=PrismaDBExceptionHandler.service_unavailable_proxy_exception(exc) + ) verbose_proxy_logger.exception("Unhandled exception in request: %s", type(exc).__name__) + if should_report_bug(exc): + verbose_proxy_logger.error( + bug_report_notice( + build_proxy_bug_report( + exc, + call_type=allowlisted(request.url.path, KNOWN_PROXY_ROUTES), + ) + ) + ) _close_dangling_otel_server_span(request, 500, exc=exc) return JSONResponse( status_code=500, @@ -2353,6 +2394,7 @@ app.add_middleware( # it sees prisma_client as of the first request rather than import time. sink_factory=lambda: gateway_request_accumulator if prisma_client is not None else None, ) +app.add_middleware(BudgetReservationReleaseMiddleware, release=release_unbound_budget_reservation) app.add_middleware(InFlightRequestsMiddleware) app.add_middleware(SecurityHeadersMiddleware) @@ -2477,6 +2519,13 @@ heuristic_v1_tuning_baselines: Mapping[str, str] | None = None # second ProxyConfig instance must not get its own independent lock over it. MODEL_RECONCILE_LOCK: Final = asyncio.Lock() general_settings: dict = {} +_GENERAL_SETTINGS_VIEW: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + + +def general_settings_view() -> Mapping[str, object]: + return _GENERAL_SETTINGS_VIEW.validate_python(general_settings) + + config_passthrough_endpoints: list[dict[str, Any]] | None = None log_file: Final = "api_log.json" worker_config: Final = None @@ -4806,6 +4855,22 @@ def validate_deployment_complexity_router_placement(model: Mapping[str, object]) raise ValueError(f"model {model.get('model_name', '')!r}: {violation}") +def validate_deployment_access_windows(model: Mapping[str, object]) -> None: + """ + Reject a malformed `model_info.access_windows` instead of silently dropping the deployment. + + Checked here rather than on `ModelInfo` because the proxy builds its router with + `ignore_invalid_deployments=True`, so a rejection further down turns a bad + deployment into a silently missing model instead of a refusal to start. + """ + model_info: Final = model.get("model_info") + if not isinstance(model_info, Mapping): + return + error: Final = access_windows_config_error(model_info, model_name=str(model.get("model_name", ""))) + if error is not None: + raise ValueError(error) + + def validate_auto_router_capability_limits(model_list: Sequence[Mapping[str, object]], *, limit: int | None) -> None: """ Refuse to start when config.yaml defines more auto-routers claiming a licensed capability than allowed. @@ -6202,6 +6267,8 @@ class ProxyConfig: litellm.upperbound_key_generate_params = LiteLLM_UpperboundKeyGenerateParams(**value) else: raise Exception(f"Invalid value set for upperbound_key_generate_params - value={value}") + elif key == "key_alias_pattern": + litellm.key_alias_pattern = parse_key_alias_pattern(value) elif key == "json_logs" and value is True: litellm.json_logs = True litellm._turn_on_json() @@ -6506,6 +6573,7 @@ class ProxyConfig: model["litellm_params"][k] = get_secret(v) validate_deployment_max_agentic_loops(model) validate_deployment_complexity_router_placement(model) + validate_deployment_access_windows(model) pin_complexity_router_model_id(model) complexity_router_config = model["litellm_params"].get("complexity_router_config") if isinstance(complexity_router_config, dict): @@ -12467,9 +12535,9 @@ async def realtime_websocket_endpoint( user_model=user_model, ) await llm_call - except websockets.exceptions.InvalidStatusCode as e: + except websockets.exceptions.InvalidStatus as e: verbose_proxy_logger.exception("Invalid status code") - await websocket.close(code=e.status_code, reason="Invalid status code") + await close_after_upstream_handshake_refusal(websocket, e.response.status_code) except Exception as e: verbose_proxy_logger.exception("Internal server error") redacted_error: Final = _redact_string(str(e)) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index f9e5c4ff1e4..e28fa2c06a4 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -36,10 +36,10 @@ from litellm.proxy.common_utils.user_api_key_cache import ( tag_cache_key, team_membership_reservation_cache_key, ) +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router -from litellm.rust_bridge.token_counter import RustTokenizer, count_input_tokens, rust_tokenizer from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget from litellm.types.router import DeploymentTypedDict @@ -366,6 +366,7 @@ async def reserve_budget_for_request( "reserved_cost": reservation_cost, "entries": applied_entries, "finalized": False, + "callback_bound": False, "input_cost": min(float(input_cost or 0.0), reservation_cost), "input_tokens": max(input_token_counts.values(), default=None), } @@ -474,6 +475,19 @@ async def release_or_invalidate_budget_reservation( budget_reservation["finalized"] = True +async def release_unbound_budget_reservation(budget_reservation: Mapping[str, object]) -> None: + """Release a reservation no logging callback took ownership of, once the request ended. + + A handler whose litellm call never builds a logging object (batch cancel, file + content, anything without the client decorator) runs no cost callback, so nothing + else would ever reconcile its reservation. A bound reservation is left alone: its + success or failure handler settles it, possibly after the response has been sent. + """ + if not isinstance(budget_reservation, dict) or budget_reservation.get("callback_bound") is True: + return + await release_or_invalidate_budget_reservation(budget_reservation=budget_reservation) + + async def _get_budget_counters( request_body: dict, valid_token: UserAPIKeyAuth, @@ -1475,9 +1489,6 @@ def _get_request_models( return (model,) if isinstance(model, str) else tuple(model) -TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000 - - async def count_request_input_tokens( request_body: dict, route: str, @@ -1486,105 +1497,11 @@ async def count_request_input_tokens( ) -> Mapping[str, int]: """Input-token count per candidate model, counted once per request. - Tokenizing is the reservation path's dominant CPU cost and is O(prompt), so - counting a large prompt inline stalls every other request on the worker. - Models whose tokenizer the Rust bridge ports (Anthropic, tiktoken cl100k_base - and o200k_base) are counted from the raw body by the bridge when it is enabled, once per - distinct tokenizer, which parses and tokenizes with the GIL released. - Everything it declines is counted in Python, large prompts in a worker - thread. The counts are reused by both the max-cost and the input-cost - estimate. - """ + The counts are reused by both the max-cost and the input-cost estimate.""" models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router) if not models: return MappingProxyType({}) - tokenizers: Final[Mapping[str, RustTokenizer | None]] = MappingProxyType( - {model: rust_tokenizer(model) for model in models} - ) - distinct_tokenizers: Final[tuple[RustTokenizer, ...]] = tuple( - dict.fromkeys(tokenizer for tokenizer in tokenizers.values() if tokenizer is not None) - ) - rust_counts_by_tokenizer: Final[Mapping[RustTokenizer, int]] = MappingProxyType( - { - tokenizer: count.input_tokens - for tokenizer in distinct_tokenizers - if raw_body is not None and (count := await count_input_tokens(raw_body, tokenizer)) is not None - } - ) - rust_counts: Final = MappingProxyType( - { - model: rust_counts_by_tokenizer[tokenizer] - for model, tokenizer in tokenizers.items() - if tokenizer is not None and tokenizer in rust_counts_by_tokenizer - } - ) - python_models: Final = tuple(model for model in models if model not in rust_counts) - python_counts: Final = ( - MappingProxyType({}) - if not python_models - else _count_input_tokens_for_models(request_body=request_body, models=python_models) - if _approximate_input_size(request_body) < TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS - else await asyncio.to_thread( - _count_input_tokens_for_models, - request_body=request_body, - models=python_models, - ) - ) - verbose_proxy_logger.debug("input token counts: rust=%s python=%s", dict(rust_counts), dict(python_counts)) - return MappingProxyType({**rust_counts, **python_counts}) - - -def _count_input_tokens_for_models( - request_body: dict, - models: Sequence[str], -) -> Mapping[str, int]: - return MappingProxyType( - { - model: tokens - for model in models - if (tokens := _count_input_tokens(request_body=request_body, model=model)) is not None - } - ) - - -_INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice") - - -def _approximate_input_size(request_body: Mapping[str, object]) -> int: - """Length of the request's input text, a cheap stand-in for tokenizing cost. - - Every field _count_input_tokens hands the tokenizer is sized here, and - rendering rather than walking keeps mapping keys in the total, which a tool - schema's property names are.""" - return sum(len(str(request_body.get(field, ""))) for field in _INPUT_SIZE_FIELDS) - - -def _count_input_tokens(request_body: dict, model: str) -> int | None: - try: - if "messages" in request_body: - try: - return litellm.token_counter( - model=model, - messages=request_body.get("messages") or (), - tools=request_body.get("tools"), - tool_choice=request_body.get("tool_choice"), - ) - except ValueError: - return _count_text_tokens(model=model, text=request_body.get("messages")) - if "prompt" in request_body: - return _count_text_tokens(model=model, text=request_body.get("prompt")) - if "input" in request_body: - return _count_text_tokens(model=model, text=request_body.get("input")) - if "query" in request_body or "documents" in request_body: - query_tokens: Final = _count_text_tokens(model=model, text=request_body.get("query")) - document_tokens: Final = _count_text_tokens( - model=model, - text=request_body.get("documents"), - ) - return query_tokens + document_tokens - except Exception: - verbose_proxy_logger.debug("Unable to count input tokens for budget reservation", exc_info=True) - return None + return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models) def _estimate_input_tokens( @@ -1595,7 +1512,9 @@ def _estimate_input_tokens( input_tokens: int | None = None, ) -> int | None: counted: Final = ( - input_tokens if input_tokens is not None else _count_input_tokens(request_body=request_body, model=model) + input_tokens + if input_tokens is not None + else count_input_tokens_for_model(request_body=request_body, model=model) ) if counted is not None: return counted @@ -1644,26 +1563,6 @@ def _requested_output_tokens(request_body: Mapping[str, object]) -> int | None: return next((tokens for tokens in map(_to_int, candidates) if tokens is not None), None) -def _count_text_tokens(model: str, text: object) -> int: - if text is None: - return 0 - - token_count = 0 - stack: Final = [text] - while stack: - item = stack.pop() - if item is None: - continue - if isinstance(item, list): - stack.extend(item) - continue - if isinstance(item, dict): - token_count += litellm.token_counter(model=model, text=json.dumps(item)) - continue - token_count += litellm.token_counter(model=model, text=str(item)) - return token_count - - def _get_output_multiplier(request_body: dict) -> int: output_multiplier = 1 for key in ("n", "best_of"): diff --git a/litellm/proxy/spend_tracking/input_tokens.py b/litellm/proxy/spend_tracking/input_tokens.py new file mode 100644 index 00000000000..6c7083fb6db --- /dev/null +++ b/litellm/proxy/spend_tracking/input_tokens.py @@ -0,0 +1,173 @@ +"""Input-token counting for the budget reservation path. + +Tokenizing is the reservation path's dominant CPU cost and is O(prompt), so +counting a large prompt inline stalls every other request on the worker. +Models whose tokenizer the Rust bridge ports (Anthropic, tiktoken cl100k_base +and o200k_base) are counted from the raw body by the bridge, once per distinct +tokenizer, which parses and tokenizes with the GIL released. Everything it +declines, and every model with no Rust tokenizer, is counted in Python, large +prompts in a worker thread. +""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Final + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.rust_bridge import runtime +from litellm.rust_bridge.catalog import Route, RouteContext +from litellm.rust_bridge.token_counter import ( + TOKEN_COUNTER, + RustTokenCounterFactory, + RustTokenizer, + native_count, + rust_tokenizer, +) + +TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000 + +_INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice") + + +def _approximate_input_size(request_body: Mapping[str, object]) -> int: + """Length of the request's input text, a cheap stand-in for tokenizing cost. + + Every field count_input_tokens_for_model hands the tokenizer is sized here, + and rendering rather than walking keeps mapping keys in the total, which a + tool schema's property names are.""" + return sum(len(str(request_body.get(field, ""))) for field in _INPUT_SIZE_FIELDS) + + +async def count_input_tokens( + request_body: dict, + raw_body: bytes | None, + models: Sequence[str], +) -> Mapping[str, int]: + """Input-token count per model, sharing one native count across models that + select the same tokenizer.""" + tokenizers: Final[tuple[tuple[str, RustTokenizer | None], ...]] = tuple( + (model, rust_tokenizer(model)) for model in models + ) + groups: Final[tuple[RustTokenizer | None, ...]] = tuple(dict.fromkeys(tokenizer for _, tokenizer in tokenizers)) + group_counts: Final = [ + await _count_group( + request_body=request_body, + raw_body=raw_body, + tokenizer=tokenizer, + models=tuple(model for model, selected in tokenizers if selected == tokenizer), + ) + for tokenizer in groups + ] + counts: Final = MappingProxyType({model: tokens for group in group_counts for model, tokens in group.items()}) + verbose_proxy_logger.debug("input token counts: %s", dict(counts)) + return counts + + +async def _count_group( + request_body: dict, + raw_body: bytes | None, + tokenizer: RustTokenizer | None, + models: tuple[str, ...], +) -> Mapping[str, int]: + async def python() -> Mapping[str, int]: + if _approximate_input_size(request_body) < TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: + return _count_input_tokens_for_models(request_body=request_body, models=models) + return await asyncio.to_thread( + _count_input_tokens_for_models, + request_body=request_body, + models=models, + ) + + if tokenizer is None or raw_body is None: + return await python() + try: + return await runtime.arun( + RouteContext(Route.TOKEN_COUNTER, provider=tokenizer), + binding=TOKEN_COUNTER, + native=lambda factory: _native_counts(factory, tokenizer, raw_body, models), + python=python, + ) + except (RuntimeError, ValueError) as error: + from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted, ProcessReservedForForking + + if isinstance(error, (ForkedAfterNativeRuntimeStarted, ProcessReservedForForking)): + raise + verbose_proxy_logger.debug("Rust token counter (%s) failed, counting in Python: %s", tokenizer, error) + return await python() + + +async def _native_counts( + factory: RustTokenCounterFactory, + tokenizer: RustTokenizer, + raw_body: bytes, + models: tuple[str, ...], +) -> Mapping[str, int]: + count: Final = await native_count(factory, tokenizer, raw_body) + verbose_proxy_logger.debug("Rust token counter (%s) counted %d input tokens", tokenizer, count.input_tokens) + return MappingProxyType({model: count.input_tokens for model in models}) + + +def _count_input_tokens_for_models( + request_body: dict, + models: Sequence[str], +) -> Mapping[str, int]: + return MappingProxyType( + { + model: tokens + for model in models + if (tokens := count_input_tokens_for_model(request_body=request_body, model=model)) is not None + } + ) + + +def count_input_tokens_for_model(request_body: dict, model: str) -> int | None: + try: + if "messages" in request_body: + try: + return litellm.token_counter( + model=model, + messages=request_body.get("messages") or (), + tools=request_body.get("tools"), + tool_choice=request_body.get("tool_choice"), + ) + except ValueError: + return _count_text_tokens(model=model, text=request_body.get("messages")) + if "prompt" in request_body: + return _count_text_tokens(model=model, text=request_body.get("prompt")) + if "input" in request_body: + return _count_text_tokens(model=model, text=request_body.get("input")) + if "query" in request_body or "documents" in request_body: + query_tokens: Final = _count_text_tokens(model=model, text=request_body.get("query")) + document_tokens: Final = _count_text_tokens( + model=model, + text=request_body.get("documents"), + ) + return query_tokens + document_tokens + except Exception: + verbose_proxy_logger.debug("Unable to count input tokens for budget reservation", exc_info=True) + return None + + +def _count_text_tokens(model: str, text: object) -> int: + if text is None: + return 0 + + token_count = 0 + stack: Final = [text] + while stack: + item = stack.pop() + if item is None: + continue + if isinstance(item, list): + stack.extend(item) + continue + if isinstance(item, dict): + token_count += litellm.token_counter(model=model, text=json.dumps(item)) + continue + token_count += litellm.token_counter(model=model, text=str(item)) + return token_count diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index a319535f725..b5980f9b224 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -69,6 +69,18 @@ _SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)" _SESSION_GROUP_KEY_SQL: Final = f"{_SESSION_KEY_EXPR}, api_key" _MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')" _AGENT_CALL_TYPE_SQL: Final = "'asend_message'" +_BATCH_CALL_TYPES_SQL: Final = "('acreate_batch', 'create_batch', 'aretrieve_batch', 'retrieve_batch')" +_SPAN_TYPE_SQL_CONDITIONS: Final[Mapping[str, str]] = MappingProxyType( + { + "mcp": f"call_type IN {_MCP_CALL_TYPES_SQL}", + "agent": f"call_type = {_AGENT_CALL_TYPE_SQL}", + "batch": f"call_type IN {_BATCH_CALL_TYPES_SQL}", + "llm": ( + f"(call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL} " + f"AND call_type NOT IN {_BATCH_CALL_TYPES_SQL})" + ), + } +) _SPEND_LOG_LIST_COLUMNS: Final = """ request_id, call_type, api_key, spend, total_tokens, prompt_tokens, completion_tokens, "startTime", "endTime", @@ -2410,6 +2422,10 @@ async def ui_view_spend_logs( default=None, description="Filter logs by cache state: 'hit' or 'miss'. Miss includes legacy rows with a null/unknown cache state", ), + span_type: str | None = fastapi.Query( + default=None, + description="Filter logs by span type: llm, agent, mcp, or batch", + ), model: str | None = fastapi.Query(default=None, description="Filter logs by model"), model_id: str | None = fastapi.Query( default=None, @@ -2512,6 +2528,13 @@ async def ui_view_spend_logs( param="cache_hit_filter", code=status.HTTP_400_BAD_REQUEST, ) + if isinstance(span_type, str) and span_type not in _SPAN_TYPE_SQL_CONDITIONS: + raise ProxyException( + message=f"Invalid span_type: {span_type}. Must be one of: llm, agent, mcp, batch", + type="bad_request", + param="span_type", + code=status.HTTP_400_BAD_REQUEST, + ) try: is_admin_view: Final = _is_admin_view_safe(user_api_key_dict=user_api_key_dict) @@ -2776,6 +2799,10 @@ async def ui_view_spend_logs( elif cache_hit_filter == "miss": sql_conditions.append("(cache_hit IS NULL OR LOWER(cache_hit) != 'true')") + span_type_condition: Final = _span_type_sql_condition(span_type) + if span_type_condition is not None: + sql_conditions.append(span_type_condition) + if exclude_internal_health_checks: sql_conditions.append(f"api_key NOT IN (${p}, ${p + 1})") sql_params.extend(_INTERNAL_HEALTH_CHECK_API_KEYS) @@ -4673,6 +4700,12 @@ def _build_status_filter_condition(status_filter: str | None) -> Mapping[str, ob return {"status": {"equals": status_filter}} +def _span_type_sql_condition(span_type: str | None) -> str | None: + if span_type is None: + return None + return _SPAN_TYPE_SQL_CONDITIONS.get(span_type) + + def _is_admin_view_safe(user_api_key_dict: UserAPIKeyAuth) -> bool: """ Safely determine if the current user has admin view permissions. diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index f4d4ccf5851..d520177965c 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -364,11 +364,18 @@ ALLOWED_UI_SETTINGS_FIELDS: Final = { } ENABLE_PTU_COST_ATTRIBUTION_UI_SETTING: Final = "enable_ptu_cost_attribution" +APPLY_USER_BUDGET_TO_TEAM_KEYS_UI_SETTING: Final = "apply_user_budget_to_team_keys" # UI settings derived from the deployment environment. Deliberately kept out of # ALLOWED_UI_SETTINGS_FIELDS: they are read-only, never persisted, and PATCH # rejects them so an admin cannot flip an env-gated feature at runtime. -_DERIVED_UI_SETTINGS_FIELDS: Final[frozenset[str]] = frozenset({ENABLE_PTU_COST_ATTRIBUTION_UI_SETTING}) +_DERIVED_UI_SETTINGS_FIELDS: Final[frozenset[str]] = frozenset( + {ENABLE_PTU_COST_ATTRIBUTION_UI_SETTING, APPLY_USER_BUDGET_TO_TEAM_KEYS_UI_SETTING} +) + + +def _apply_user_budget_to_team_keys_enabled(settings: Mapping[str, object]) -> bool: + return settings.get(APPLY_USER_BUDGET_TO_TEAM_KEYS_UI_SETTING) is True def _derived_ui_setting_value(key: str) -> object: @@ -381,6 +388,12 @@ def _derived_ui_setting_value(key: str) -> object: """ if key == ENABLE_PTU_COST_ATTRIBUTION_UI_SETTING: return is_ptu_cost_attribution_enabled() + if key == APPLY_USER_BUDGET_TO_TEAM_KEYS_UI_SETTING: + from litellm.proxy.proxy_server import general_settings + + return _apply_user_budget_to_team_keys_enabled( + cast(Mapping[str, object], general_settings) # cast-ok: proxy_server declares general_settings as bare dict + ) return None @@ -394,6 +407,7 @@ _RUNTIME_GENERAL_SETTINGS_FLAGS: Final = [ "allow_agents_for_team_admins", "disable_vector_stores_for_internal_users", "allow_vector_stores_for_team_admins", + "disable_custom_api_keys", "disable_key_generate_for_org_admin", TEAM_ADMIN_EDITABLE_TEAM_FIELDS_SETTING, ] @@ -776,6 +790,8 @@ def _ui_setting_source( if key == ENABLE_PTU_COST_ATTRIBUTION_UI_SETTING: configured_value: Final = get_secret(PTU_COST_ATTRIBUTION_ENV_VAR, None) return "config" if configured_value is not None or value is True else "default" + if key == APPLY_USER_BUDGET_TO_TEAM_KEYS_UI_SETTING: + return "config" if value is True else "default" return source_for(settings, key, _model_field_default(settings_class, key)) @@ -1782,6 +1798,9 @@ async def get_ui_settings(): { **resolved_settings.values, ENABLE_PTU_COST_ATTRIBUTION_UI_SETTING: is_ptu_cost_attribution_enabled(), + APPLY_USER_BUDGET_TO_TEAM_KEYS_UI_SETTING: _derived_ui_setting_value( + APPLY_USER_BUDGET_TO_TEAM_KEYS_UI_SETTING + ), } ) source: Final[Mapping[str, FieldSource]] = MappingProxyType( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index bfd412db1a7..c7ce9d081d6 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -52,11 +52,17 @@ from litellm.constants import ( DEFAULT_MODEL_CREATED_AT_TIME, LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, MAX_TEAM_LIST_LIMIT, + PROXY_REJECTED_BEFORE_ROUTING_KEY, REDIS_SPEND_LOGS_BUFFER_DEQUEUE_COUNT, SPEND_LOG_QUEUE_MAX_BYTES, SPEND_LOG_WRITE_BATCH_MAX_BYTES, SPEND_LOG_WRITE_BATCH_MAX_ROWS, ) +from litellm.litellm_core_utils.bug_report import ( + bug_report_notice, + should_report_bug, + strip_bug_report_notice, +) from litellm.proxy._types import ( CommonProxyErrors, ProxyErrorTypes, @@ -64,6 +70,7 @@ from litellm.proxy._types import ( SpendLogsMetadata, SpendLogsPayload, ) +from litellm.proxy.bug_report_config import build_proxy_bug_report from litellm.proxy.common_utils.openai_error_payload import ( litellm_call_id_headers, openai_error_param, @@ -138,6 +145,10 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads +from litellm.litellm_core_utils.served_output_texts import ( + record_served_output_texts, + served_stream_output_texts, +) from litellm.litellm_core_utils.token_counter import offload_token_count from litellm.llms import load_guardrail_translation_mappings from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -969,6 +980,92 @@ def _failure_usage_to_lift( _EMPTY_LIFT: Final = MappingProxyType({}) +def _stamp_deployment_attribution( + litellm_params: dict[str, object], model_group: str | None, team_id: str | None, dispatched: bool +) -> Mapping[str, object]: + """Stamp provider and logging-metadata attribution onto ``litellm_params`` and return it. + ``litellm_params["model_info"]`` stays unset: the router's cooldown and per-deployment rpm + callbacks key off it and must not count a proxy-side reject against the deployment. A failure + after the provider handoff keeps the metadata the router stamped; a request that never reached a + provider is flagged ``PROXY_REJECTED_BEFORE_ROUTING_KEY`` (deployment metrics key off it) whatever + its metadata says, since ``metadata.model_info`` can be caller supplied.""" + attribution: Final = _deployment_attribution_for_model_group(model_group, team_id) + if "custom_llm_provider" in attribution: + litellm_params["custom_llm_provider"] = attribution["custom_llm_provider"] + if dispatched: + return attribution + litellm_params[PROXY_REJECTED_BEFORE_ROUTING_KEY] = True + if "model_info" not in attribution: + return attribution + if litellm_params.get("metadata") is None: + litellm_params["metadata"] = {} # mutable-ok: legacy logging payload is populated in place + metadata: Final = litellm_params["metadata"] + if not isinstance(metadata, dict): + return attribution + metadata.setdefault("model_info", attribution["model_info"]) + metadata.setdefault("deployment", attribution["deployment"]) + return attribution + + +def _deployment_attribution_for_model_group(model_group: object, team_id: str | None) -> Mapping[str, object]: + """Provider fields the router would have stamped had it reached a deployment: + ``custom_llm_provider`` when every deployment in the group resolves to the same + provider, plus ``model_info`` and ``deployment`` when the group has exactly one. + ``team_id`` picks the key's team deployments over a global group of the same public name.""" + if not isinstance(model_group, str): + return _EMPTY_LIFT + + from litellm.proxy.proxy_server import llm_router + + if llm_router is None: + return _EMPTY_LIFT + deployments: Final = llm_router.get_model_list(model_name=model_group, team_id=team_id) + if not deployments: + return _EMPTY_LIFT + + def _provider_for_deployment(deployment: Mapping[str, object]) -> str | None: + litellm_params: Final = cast( # cast-ok: router deployment parameters are mapping-shaped + Mapping[str, object], deployment["litellm_params"] + ) + try: + provider: Final = litellm.get_llm_provider( + model=cast(str, litellm_params["model"]), # cast-ok: router deployment model is a string + custom_llm_provider=cast( # cast-ok: router deployment provider is optional + str | None, litellm_params.get("custom_llm_provider") + ), + )[1] + return cast(str | None, provider) # cast-ok: provider resolver returns an optional provider string + except Exception: # noqa: BLE001 # get_llm_provider raises for unmapped models + return None + + providers: Final = frozenset(_provider_for_deployment(deployment) for deployment in deployments) + shared_provider: Final = next(iter(providers)) if len(providers) == 1 else None + single_deployment: Final = deployments[0] if len(deployments) == 1 else None + single_deployment_params: Final = ( + cast( # cast-ok: router deployment parameters are mapping-shaped + Mapping[str, object], single_deployment["litellm_params"] + ) + if single_deployment is not None + else None + ) + return MappingProxyType( + { + # mutable-ok: frozen immediately by the outer MappingProxyType + **({"custom_llm_provider": shared_provider} if shared_provider is not None else {}), + **( + { # mutable-ok: frozen immediately by the outer MappingProxyType + "model_info": dict( # mutable-ok: preserve the router's mutable model-info payload + single_deployment.get("model_info") or {} + ), + "deployment": single_deployment_params["model"], + } + if single_deployment is not None and single_deployment_params is not None + else {} # mutable-ok: frozen immediately by the outer MappingProxyType + ), + } + ) + + def _call_type_for_route(route: str | None) -> str | None: """The route's call type when it maps to a single operation (its async and sync variants); None for routes shared by several operations, since the method is not known here.""" @@ -3217,11 +3314,25 @@ class ProxyLogging: elif k not in ("model", "user", "litellm_logging_obj"): _optional_params[k] = v + attribution: Final = _stamp_deployment_attribution( + _litellm_params, + request_data.get("model"), + user_api_key_dict.team_id, + dispatched=litellm_logging_obj.model_call_details.get("first_api_call_start_time") is not None, + ) + litellm_logging_obj.update_environment_variables( model=request_data.get("model", ""), user=request_data.get("user", ""), optional_params=_optional_params, litellm_params=_litellm_params, + **( + { # mutable-ok: frozen immediately by keyword expansion + "custom_llm_provider": attribution["custom_llm_provider"] + } + if "custom_llm_provider" in attribution + else {} # mutable-ok: frozen immediately by keyword expansion + ), ) input: list | str | dict = "" @@ -3803,12 +3914,16 @@ class ProxyLogging: translation=pipeline_translation, ) + served_chunks: Final[list[object]] = [] # mutable-ok: accumulates while yielding to the client try: async for chunk in current_response: + served_chunks.append(chunk) yield chunk except (GeneratorExit, asyncio.CancelledError): + ProxyLogging._record_served_stream_output(request_data, served_chunks) raise except Exception as e: + ProxyLogging._record_served_stream_output(request_data, served_chunks) if not ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, e): ProxyLogging._fire_deferred_stream_logging(request_data) raise @@ -3817,6 +3932,7 @@ class ProxyLogging: # completed. unified_guardrail writes guardrail_information during # its end-of-stream block (inside current_response), so by the time # we reach this point the metadata is fully populated. + ProxyLogging._record_served_stream_output(request_data, served_chunks) ProxyLogging._fire_deferred_stream_logging(request_data) async def _pipeline_gated_stream( @@ -3884,6 +4000,13 @@ class ProxyLogging: for buffered_item in buffered: yield buffered_item + @staticmethod + def _record_served_stream_output(request_data: Mapping[str, object], served_chunks: Sequence[object]) -> None: + logging_obj: Final = request_data.get("litellm_logging_obj") + if not isinstance(logging_obj, Logging): + return + record_served_output_texts(logging_obj.model_call_details, served_stream_output_texts(served_chunks)) + @staticmethod def _fire_deferred_stream_logging(request_data: dict) -> None: """ @@ -7989,8 +8112,10 @@ def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None) elif isinstance(e, ProxyException): return with_litellm_call_id(e, litellm_call_id) _status_code: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR) + if should_report_bug(e): + verbose_proxy_logger.error(bug_report_notice(build_proxy_bug_report(e))) return ProxyException( - message=str(e), + message=strip_bug_report_notice(str(e)), type=ProxyErrorTypes.internal_server_error, param=openai_error_param(e), headers=headers, diff --git a/litellm/responses/dispatch.py b/litellm/responses/dispatch.py index b2748fca4b6..d240356805c 100644 --- a/litellm/responses/dispatch.py +++ b/litellm/responses/dispatch.py @@ -5,7 +5,7 @@ from typing import Final, TypeAlias, cast # noqa: TID251 # native binding sele from litellm.responses import main from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator -from litellm.rust_bridge.catalog import Context, Delivery, Route +from litellm.rust_bridge.catalog import Delivery, Route, RouteContext from litellm.rust_bridge.dispatch import PublicDispatch, call_hook from litellm.rust_bridge.public_call import bind, optional_bool, optional_mapping, optional_str, signature from litellm.rust_bridge.responses.entrypoints import ( @@ -64,8 +64,8 @@ def _public_request( ) -def _context(request: LiteLLMResponsesRequest) -> Context: - return Context( +def _context(request: LiteLLMResponsesRequest) -> RouteContext: + return RouteContext( Route.RESPONSES, provider=request.custom_llm_provider, model=request.model, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 5a4a08b760c..e771cdf5ae4 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -26,6 +26,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( update_responses_input_with_model_file_ids, update_responses_tools_with_model_file_ids, ) +from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig @@ -208,16 +209,12 @@ async def aresponses_api_with_mcp( user_api_key_auth = kwargs.get("user_api_key_auth") or kwargs.get("litellm_metadata", {}).get("user_api_key_auth") # Extract MCP auth headers from request (for dynamic auth when fetching tools) - mcp_auth_header: str | None = None - mcp_server_auth_headers: dict[str, dict[str, str]] | None = None - secret_fields = kwargs.get("secret_fields") - if secret_fields and isinstance(secret_fields, dict): - ( - mcp_auth_header, - mcp_server_auth_headers, - _, - _, - ) = ResponsesAPIRequestUtils.extract_mcp_headers_from_request(secret_fields=secret_fields, tools=tools) + secret_fields: Final = kwargs.get("secret_fields") + mcp_auth_header, mcp_server_auth_headers, _, discovery_raw_headers = ( + ResponsesAPIRequestUtils.extract_mcp_headers_from_request(secret_fields=secret_fields, tools=tools) + if isinstance(secret_fields, dict) and secret_fields + else (None, None, None, None) + ) # Get original MCP tools (for events) and OpenAI tools (for LLM) by reusing existing methods ( @@ -230,6 +227,7 @@ async def aresponses_api_with_mcp( mcp_auth_header=mcp_auth_header, mcp_server_auth_headers=mcp_server_auth_headers, request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs), + raw_headers=discovery_raw_headers, ) openai_tools: Final = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai(original_mcp_tools) @@ -329,7 +327,6 @@ async def aresponses_api_with_mcp( user_api_key_auth = kwargs.get("litellm_metadata", {}).get("user_api_key_auth") # Extract MCP auth headers from the request to pass to MCP server - secret_fields = kwargs.get("secret_fields") ( mcp_auth_header, mcp_server_auth_headers, @@ -415,6 +412,7 @@ async def aresponses_api_with_mcp( mcp_auth_header=mcp_auth_header, mcp_server_auth_headers=mcp_server_auth_headers, request_tags=LiteLLM_Proxy_MCP_Handler._get_parent_request_tags(kwargs), + raw_headers=discovery_raw_headers, ) final_response = LiteLLM_Proxy_MCP_Handler._add_mcp_output_elements_to_response( response=final_response, @@ -433,13 +431,18 @@ def _bridges_to_chat_completions( return responses_api_provider_config is None or use_chat_completions_api is True +_RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED: Final = frozenset({"client_metadata"}) + + def _bridge_kwargs( kwargs: Mapping[str, object], responses_api_provider_config: BaseResponsesAPIConfig | None, allowed_openai_params: Sequence[str] | None, ) -> Mapping[str, object]: if responses_api_provider_config is None: - return kwargs + return MappingProxyType( + {key: value for key, value in kwargs.items() if key not in _RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED} + ) forwarded_keys: Final = frozenset( ( *litellm.OPENAI_CHAT_COMPLETION_PARAMS, @@ -448,7 +451,7 @@ def _bridge_kwargs( *GenericLiteLLMParams.model_fields, *(allowed_openai_params or ()), ) - ) + ).difference(_RESPONSES_ONLY_REQUEST_FIELDS_NEVER_BRIDGED) return MappingProxyType({key: value for key, value in kwargs.items() if key in forwarded_keys}) @@ -1213,6 +1216,27 @@ def responses( # get llm provider logic litellm_params: Final = GenericLiteLLMParams(**kwargs) + try: + effective_extra_headers: Final = ( + add_provider_affinity_header( + headers=extra_headers or MappingProxyType({}), + litellm_params=MappingProxyType( + { + "provider_affinity_header": litellm_params.provider_affinity_header, + "litellm_session_id": kwargs.get("litellm_session_id"), + "session_id": kwargs.get("session_id"), + "metadata": metadata, + "litellm_metadata": kwargs.get("litellm_metadata"), + } + ), + ) + if litellm_params.provider_affinity_header is not None + else extra_headers + ) + except ValueError as affinity_error: + raise litellm.BadRequestError( + message=str(affinity_error), model=model, llm_provider=custom_llm_provider + ) from affinity_error ######################################################### # MOCK RESPONSE LOGIC @@ -1256,7 +1280,7 @@ def responses( top_p=top_p, truncation=truncation, user=user, - extra_headers=extra_headers, + extra_headers=effective_extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout, @@ -1327,7 +1351,7 @@ def responses( safety_identifier=safety_identifier, text_format=text_format, allowed_openai_params=allowed_openai_params, - extra_headers=extra_headers, + extra_headers=effective_extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout, @@ -1347,7 +1371,7 @@ def responses( custom_llm_provider=custom_llm_provider, _is_async=_is_async, stream=stream, - extra_headers=extra_headers, + extra_headers=effective_extra_headers, extra_body=extra_body, timeout=timeout if timeout is not None else request_timeout, allowed_openai_params=allowed_openai_params, @@ -1376,6 +1400,7 @@ def responses( "model_info": kwargs.get("model_info"), "data_residency": infer_openai_data_residency(custom_llm_provider, litellm_params.api_base), "metadata": (kwargs["litellm_metadata"] if "litellm_metadata" in kwargs else kwargs.get("metadata")), + "provider_affinity_header": litellm_params.provider_affinity_header, }, custom_llm_provider=custom_llm_provider, ) @@ -1395,7 +1420,7 @@ def responses( custom_llm_provider=custom_llm_provider, litellm_params=litellm_params, logging_obj=litellm_logging_obj, - extra_headers=extra_headers, + extra_headers=effective_extra_headers, extra_body=extra_body, timeout=timeout or request_timeout, _is_async=_is_async, diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index df1e3e62441..b17e0befba6 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -138,6 +138,7 @@ async def acompletion_with_mcp( mcp_auth_header=mcp_auth_header, mcp_server_auth_headers=mcp_server_auth_headers, request_tags=request_tags, + raw_headers=raw_headers, ) openai_tools: Final = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai( diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 88a2b92c680..71f61079154 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -236,6 +236,7 @@ class LiteLLM_Proxy_MCP_Handler: mcp_auth_header: str | None = None, mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, request_tags: list[str] | None = None, + raw_headers: dict[str, str] | None = None, ) -> tuple[list[MCPTool], list[str]]: """ Get available tools from the MCP server manager. @@ -326,6 +327,7 @@ class LiteLLM_Proxy_MCP_Handler: list_tools_log_source="responses", litellm_trace_id=litellm_trace_id, request_tags=request_tags, + raw_headers=raw_headers, ) tools: Final = listing.tools @@ -452,6 +454,7 @@ class LiteLLM_Proxy_MCP_Handler: mcp_auth_header: str | None = None, mcp_server_auth_headers: dict[str, dict[str, str]] | None = None, request_tags: list[str] | None = None, + raw_headers: dict[str, str] | None = None, ) -> tuple[list[MCPTool], dict[str, str]]: """ Process MCP tools through filtering and deduplication pipeline without OpenAI transformation. @@ -482,6 +485,7 @@ class LiteLLM_Proxy_MCP_Handler: mcp_auth_header=mcp_auth_header, mcp_server_auth_headers=mcp_server_auth_headers, request_tags=request_tags, + raw_headers=raw_headers, ) # Step 2: Filter tools based on allowed_tools parameter @@ -872,7 +876,7 @@ class LiteLLM_Proxy_MCP_Handler: if litellm_logging_obj: try: litellm_logging_obj.post_call(original_response=result) - await litellm_logging_obj.async_post_mcp_tool_call_hook( + result = await litellm_logging_obj.async_post_mcp_tool_call_hook( kwargs=litellm_logging_obj.model_call_details, response_obj=result, start_time=start_time, diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index c60020ab979..3b5cb85862d 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -91,16 +91,6 @@ async def create_mcp_list_tools_events( # Use the pre-processed MCP tools that were already fetched, filtered, and deduplicated by the parent filtered_mcp_tools: Final = pre_processed_mcp_tools - # Convert tools to dict format for the event - _mcp_tools_dict: Final = [ - tool.model_dump() - if hasattr(tool, "model_dump") and callable(getattr(tool, "model_dump", None)) - else tool.__dict__ - if hasattr(tool, "__dict__") - else {"name": getattr(tool, "name", str(tool))} - for tool in filtered_mcp_tools - ] - # Emit list tools completed event completed_event: Final = MCPListToolsCompletedEvent( type=ResponsesAPIStreamEvents.MCP_LIST_TOOLS_COMPLETED, diff --git a/litellm/router.py b/litellm/router.py index 9a5c770e78a..7267f6eb3ba 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -31,6 +31,7 @@ from collections.abc import ( MutableMapping, Sequence, ) +from datetime import datetime, timezone from functools import lru_cache, partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeAlias, TypeVar, Union, cast @@ -118,6 +119,15 @@ from litellm.llms.openai_like.model_info import ( get_openai_compatible_model_info, ) from litellm.router_strategy.budget_limiter import RouterBudgetLimiting +from litellm.router_strategy.complexity_router.context_compaction import ( + arm_compaction, + compact_to_fit, + compaction_pending, + initialize_compaction_state, + is_native_compaction_call, + reject_recursive_compactor, + surface_for_call, +) from litellm.router_strategy.least_busy import LeastBusyLoggingHandler from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler @@ -129,12 +139,14 @@ from litellm.router_strategy.tag_based_routing import ( get_deployments_for_tag, is_valid_deployment_tag, ) +from litellm.router_utils.access_windows import access_windows_config_error, filter_reserved_deployments from litellm.router_utils.add_retry_fallback_headers import ( _HiddenParamsHost, add_fallback_headers_to_response, add_retry_headers_to_response, apply_quality_router_decision_headers, apply_remaining_usage_headers, + apply_response_model_id, complexity_router_decision_headers, ensure_response_additional_headers, get_hidden_params_dict, @@ -165,6 +177,8 @@ from litellm.router_utils.common_utils import ( _is_proxy_admin_request, filter_team_based_models, filter_web_search_deployments, + format_fallback_outcome_message, + format_no_fallback_group_message, get_request_team_id, provider_for_generic_call, resolve_model_group_alias, @@ -192,6 +206,7 @@ from litellm.router_utils.fallback_event_handlers import ( fallback_lookup_groups, fallbacks_disabled_for_request, get_fallback_model_group_for_lookup_groups, + get_pre_routing_selection, has_unattempted_fallback_target, mid_stream_fallback_hop_kwargs, per_request_fallback_controls, @@ -234,7 +249,11 @@ from litellm.router_utils.router_callbacks.track_deployment_metrics import ( increment_deployment_failures_for_current_minute, increment_deployment_successes_for_current_minute, ) -from litellm.router_utils.routing_groups import parse_routing_groups, validate_routing_strategy +from litellm.router_utils.routing_groups import ( + apply_routing_group_priority, + parse_routing_groups, + validate_routing_strategy, +) from litellm.scheduler import FlowItem, Scheduler from litellm.types.llms.openai import ( AllMessageValues, @@ -828,7 +847,9 @@ class Router: cooldown_time (float): Time to cooldown a deployment after failure in seconds. Defaults to 1. routing_strategy (Literal["simple-shuffle", "least-busy", "usage-based-routing", "latency-based-routing", "cost-based-routing"]): Routing strategy used for the implicit "default" group (any model not claimed by an entry in `routing_groups`). Defaults to "simple-shuffle". routing_strategy_args (dict): Additional args for the default group's routing strategy (e.g. latency window). Defaults to {}. - routing_groups (Optional[List[RoutingGroup]]): Named subsets of `model_name`s that use a per-group routing strategy and args. Each model belongs to at most one explicit group; everything else lands in the implicit "default" group driven by `routing_strategy` / `routing_strategy_args`. Defaults to None. + routing_groups (Optional[List[RoutingGroup]]): Named subsets of `model_name`s with a group routing strategy. + Priority groups apply only to group calls and may overlap. Other groups supply their members' default + strategy, with at most one such group per model. Unclaimed models use the top-level strategy. alerting_config (AlertingConfig): Slack alerting configuration. Defaults to None. provider_budget_config (ProviderBudgetConfig): Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None. deployment_affinity_ttl_seconds (int): TTL for user-key -> deployment affinity mapping. Defaults to 3600. @@ -1436,10 +1457,10 @@ class Router: ) -> None: """ Validates and indexes `routing_groups`. Each `model_name` may belong to - at most one explicit group. Constructs per-group strategy selectors so + at most one non-priority group. Constructs per-group strategy selectors so groups with different `routing_strategy_args` track independent state. - Models not claimed by any explicit group are served by the implicit + Models not claimed by a non-priority group are served by the implicit `"default"` group, whose selectors are the `self._logger` attributes set up in `routing_strategy_init`. """ @@ -1453,6 +1474,8 @@ class Router: alias_names: Final = frozenset(self.model_group_alias or ()) for group in groups: if group.group_name in known_model_names or group.group_name in alias_names: + if group.routing_strategy == "priority": + raise ValueError("Priority routing group names must not shadow a model or alias") verbose_router_logger.warning( "routing_groups: group_name '%s' is shadowed by an existing model_name or model_group_alias; " "the group's strategy still applies to its members, but the name is not callable until renamed.", @@ -1463,7 +1486,7 @@ class Router: ( group, self._build_strategy_selector( - strategy=group.routing_strategy, + strategy="simple-shuffle" if group.routing_strategy == "priority" else group.routing_strategy, routing_strategy_args=group.routing_strategy_args or {}, register_callbacks=False, ), @@ -1488,7 +1511,10 @@ class Router: self._routing_groups: dict[str, RoutingGroup] = {group.group_name: group for group, _ in built} self._model_to_group: dict[str, str] = { - model_name: group.group_name for group, _ in built for model_name in group.models + model_name: group.group_name + for group, _ in built + if group.routing_strategy != "priority" + for model_name in group.models } self._group_selectors: dict[str, dict[str, RouterStrategySelector]] = { group.group_name: ( @@ -1538,11 +1564,16 @@ class Router: if routing_group is None: return None return [ # mutable-ok: matches _get_all_deployments' list contract expected by downstream filters - deployment + apply_routing_group_priority(routing_group, member, deployment) for member in routing_group.models for deployment in self._get_all_deployments(model_name=member, team_id=team_id) ] + def _is_priority_routing_group(self, model: str) -> bool: + resolved: Final = self._get_model_from_alias(model=model) or model + group: Final = self.get_routing_group(resolved) + return group is not None and group.routing_strategy == "priority" + def is_recognized_model(self, model: str) -> bool: """ Whether `model` names something this router serves directly: a @@ -1706,7 +1737,12 @@ class Router: self._bind_override_selector_to_request(override, override_selector, request_kwargs) return override, override_selector - group_name: Final = model if self.get_routing_group(model) is not None else self._model_to_group.get(model) + resolved_model: Final = self._get_model_from_alias(model=model) or model + group_name: Final = ( + resolved_model + if self.get_routing_group(resolved_model) is not None + else self._model_to_group.get(resolved_model) + ) if group_name is None: strategy = self._normalize_strategy(self.routing_strategy) attr: Final = self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.get(strategy or "") @@ -1715,6 +1751,8 @@ class Router: return strategy, selector group: Final = self._routing_groups[group_name] + if group.routing_strategy == "priority": + return "simple-shuffle", None strategy = self._normalize_strategy(group.routing_strategy) selector = self._group_selectors.get(group_name, {}).get(strategy or "") verbose_router_logger.debug("routing_group=%s model=%s strategy=%s", group_name, model, strategy) @@ -2231,6 +2269,7 @@ class Router: enable_responses_api_affinity=False, enable_session_id_affinity=False, model_group_affinity_config=self.model_group_affinity_config, + is_priority_group=self._is_priority_routing_group, ) self.optional_callbacks.append(affinity_callback) litellm.logging_callback_manager.add_litellm_callback(affinity_callback) @@ -2276,6 +2315,7 @@ class Router: enable_responses_api_affinity=enable_responses_api_affinity, enable_session_id_affinity=enable_session_id_affinity, model_group_affinity_config=self.model_group_affinity_config, + is_priority_group=self._is_priority_routing_group, ) self.optional_callbacks.append(affinity_callback) litellm.logging_callback_manager.add_litellm_callback(affinity_callback) @@ -2300,7 +2340,9 @@ class Router: ): continue if pre_call_check == "prompt_caching": - _callback = PromptCachingDeploymentCheck(cache=self.cache) + _callback = PromptCachingDeploymentCheck( + cache=self.cache, is_priority_group=self._is_priority_routing_group + ) elif pre_call_check == "router_budget_limiting": if self._get_router_deployment_budget_limiter() is not None: continue @@ -3635,6 +3677,7 @@ class Router: kwargs=kwargs, client_type="max_parallel_requests", ) + compacted_input: Final = await compact_to_fit(self, deployment, input_kwargs, "chat") async with contextlib.AsyncExitStack() as deployment_slot: if isinstance(max_parallel_requests_limit, MaxParallelRequestsLimit): deployment_slot.enter_context(max_parallel_requests_limit) @@ -3643,7 +3686,7 @@ class Router: logging_obj=logging_obj, parent_otel_span=parent_otel_span, ) - response = await litellm.acompletion(**input_kwargs) + response = await litellm.acompletion(**compacted_input) ## CHECK CONTENT FILTER ERROR ## if isinstance(response, ModelResponse): @@ -5247,8 +5290,14 @@ class Router: if custom_llm_provider is not None: response_kwargs["custom_llm_provider"] = custom_llm_provider + compacted_input: Final = await compact_to_fit( + self, + deployment, + response_kwargs, + surface_for_call(getattr(original_generic_function, "__name__", "")), + ) async with self._deployment_slot(deployment=deployment, kwargs=kwargs, parent_otel_span=parent_otel_span): - response = await original_generic_function(**response_kwargs) + response = await original_generic_function(**compacted_input) if self._should_raise_anthropic_refusal_error( model=model, @@ -7047,7 +7096,7 @@ class Router: self, exception: Exception, original_model_group: str, - all_deployments: list[DeploymentTypedDict], + all_deployments: Sequence[DeploymentTypedDict], args: tuple, kwargs: dict, input_kwargs: dict, @@ -7144,6 +7193,9 @@ class Router: # behind the router name, and fallbacks are configured per tier, not per router. lookup_groups: Final[tuple[str, ...]] = fallback_lookup_groups(kwargs, model_group) fallback_failure_exception_str = "" + no_fallback_group_explained = False + hop_depth: Final = kwargs.get("fallback_depth") + nested_fallback_hop: Final = isinstance(hop_depth, int) and hop_depth > 0 if disable_fallbacks is True or original_model_group is None: raise e @@ -7170,7 +7222,8 @@ class Router: _request_team_id: Final[str | None] = (kwargs.get("metadata", {}) or {}).get("user_api_key_team_id") # Use wildcard-aware lookup so order-based fallback also works for model # groups resolved via pattern routing (e.g. `openai/*` -> `openai/gpt-4.1-mini`). - all_deployments: Final = self.get_model_list(model_name=original_model_group, team_id=_request_team_id) or [] + order_model_group: Final = get_pre_routing_selection(kwargs) or original_model_group + all_deployments: Final = self.get_model_list(model_name=order_model_group, team_id=_request_team_id) or () _order_set: Final[set] = { litellm.utils._get_deployment_order(d) for d in all_deployments @@ -7183,7 +7236,7 @@ class Router: skip_up_to: Final = current_target if current_target is not None else order_values[0] # Build order-based fallback entries (skip already-tried levels) order_fallback_entries: Final[list] = [ - {"model": original_model_group, "_target_order": o} for o in order_values if o > skip_up_to + {"model": order_model_group, "_target_order": o} for o in order_values if o > skip_up_to ] # Get external fallbacks — handle both standard and non-standard formats external_fallback_group: list | None = None @@ -7335,8 +7388,13 @@ class Router: " -> ".join(lookup_groups), masked_fallbacks, ) - if hasattr(original_exception, "message") and litellm.expose_router_debug_in_errors: - original_exception.message += f"No fallback model group found for lookup_groups={' -> '.join(lookup_groups)}. Fallbacks={masked_fallbacks}" + if ( + hasattr(original_exception, "message") + and litellm.expose_router_debug_in_errors + and not nested_fallback_hop + ): + original_exception.message += format_no_fallback_group_message(lookup_groups, fallbacks) + no_fallback_group_explained = True raise original_exception input_kwargs.update( @@ -7367,11 +7425,16 @@ class Router: cooldown_info, ) - if hasattr(original_exception, "message") and litellm.expose_router_debug_in_errors: - # add the available fallbacks to the exception - original_exception.message += f". Received Model Group={model_group}\nAvailable Model Group Fallbacks={mask_sensitive_structure(fallback_model_group)}" - if len(fallback_failure_exception_str) > 0: - original_exception.message += f"\nError doing the fallback: {fallback_failure_exception_str}" + attempted_fallback_group: Final = input_kwargs.get("fallback_model_group") + if ( + hasattr(original_exception, "message") + and litellm.expose_router_debug_in_errors + and not no_fallback_group_explained + and not nested_fallback_hop + ): + original_exception.message += format_fallback_outcome_message( + model_group, attempted_fallback_group, fallback_failure_exception_str + ) raise original_exception @@ -7382,6 +7445,11 @@ class Router: If it fails after num_retries, fall back to another model group """ model_group: Final[str | None] = kwargs.get("model") + compaction_surface: Final = surface_for_call( + getattr(kwargs.get("original_generic_function") or kwargs.get("original_function"), "__name__", "") + ) + if compaction_surface is not None: + kwargs["_context_compaction_state"] = initialize_compaction_state(kwargs, compaction_surface) clear_pre_routing_selection(kwargs) # pyright: ignore[reportUnknownArgumentType] # **kwargs is untyped at this boundary if not isinstance(kwargs.get("attempted_targets"), AttemptedFallbackTargets): _fallback_metadata_key: Final = _get_router_metadata_variable_name( @@ -8753,6 +8821,9 @@ class Router: ) if ptu_error is not None and is_ptu_cost_attribution_enabled(): raise ValueError(ptu_error) + access_windows_error: Final = access_windows_config_error(_model_info, model_name=_model_name) + if access_windows_error is not None: + raise ValueError(access_windows_error) zeroed_pricing: Final = zeroed_ptu_pricing(_model_info, _litellm_params) if config_sourced else None litellm_params: Final[LiteLLM_Params] = LiteLLM_Params( **( # pyright: ignore[reportArgumentType] # untyped merged dict; already true for every field here @@ -11151,6 +11222,9 @@ class Router: return response additional_headers: Final = ensure_response_additional_headers(response) + apply_response_model_id( + response, find_deployment_metadata(request_kwargs) if request_kwargs is not None else None + ) additional_headers["x-litellm-model-group"] = model_group apply_quality_router_decision_headers(additional_headers, request_kwargs) additional_headers.update(complexity_router_decision_headers(request_kwargs)) @@ -11641,7 +11715,14 @@ class Router: else: continue - returned_models.extend(self._get_all_deployments(model_name=_router_model_name, model_alias=model_alias)) + if (alias_group := self.get_routing_group(_router_model_name)) is not None: + returned_models.extend( + {**row, "model_name": model_alias} for row in self._materialize_routing_group_rows((alias_group,)) + ) + else: + returned_models.extend( + self._get_all_deployments(model_name=_router_model_name, model_alias=model_alias) + ) return returned_models @@ -11672,7 +11753,7 @@ class Router: def _materialize_routing_group_rows(self, groups: tuple[RoutingGroup, ...]) -> tuple[DeploymentTypedDict, ...]: return tuple( - self._as_routing_group_row(deployment) + self._as_routing_group_row(apply_routing_group_priority(group, member, deployment)) for group in groups for member in group.models for deployment in self._get_all_deployments(model_name=member, model_alias=group.group_name) @@ -11979,7 +12060,10 @@ class Router: ): _settings_to_return[var] = self.lowestlatency_logger.routing_args.json() - _settings_to_return["routing_groups"] = [group.model_dump() for group in self._routing_groups.values()] + _settings_to_return["routing_groups"] = [ + group.model_dump(exclude=frozenset(("model_priorities",)) if group.model_priorities is None else None) + for group in self._routing_groups.values() + ] return _settings_to_return def update_settings(self, **kwargs): @@ -12090,8 +12174,8 @@ class Router: def _count_pre_call_check_tokens( self, - messages: list[dict[str, str]] | None, - input: str | list | None, + messages: Sequence[Mapping[str, object]] | None, + input: str | list[object] | None, request_kwargs: Mapping[str, object] | None = None, ) -> int: """ @@ -12238,7 +12322,9 @@ class Router: _rate_limit_error = False parent_otel_span: Final = _get_parent_otel_span_from_kwargs(request_kwargs) - has_countable_input: Final = messages is not None or input is not None + has_countable_input: Final = (messages is not None or input is not None) and not compaction_pending( + request_kwargs + ) ## get model group RPM ## dt: Final = get_utc_datetime() @@ -12512,12 +12598,30 @@ class Router: request_team_id: Final = get_request_team_id(request_kwargs) # check if aliases set on litellm model alias map if specific_deployment is True: - return model, self._get_deployment_by_litellm_model(model=model) + return model, self._drop_strategy_markers( + model, + self._filter_reserved_deployments( + model=model, + healthy_deployments=self._get_deployment_by_litellm_model(model=model), + request_team_id=request_team_id, + ), + ) elif model not in self.model_names and self.has_model_id(model): deployment: Final = self.get_deployment(model_id=model) if deployment is not None: deployment_model: Final = deployment.litellm_params.model - return deployment_model, deployment.model_dump(exclude_none=True) + return deployment_model, cast( # cast-ok: contract requires a plain dict for a single deployment + dict, + self._filter_reserved_deployments( + model=deployment_model, + healthy_deployments=( + cast( # cast-ok: model_dump of a router deployment + DeploymentTypedDict, deployment.model_dump(exclude_none=True) + ), + ), + request_team_id=request_team_id, + )[0], + ) raise ValueError( f"LiteLLM Router: Trying to call specific deployment, but Model ID :{model} does not exist in Model ID map" ) @@ -12535,8 +12639,26 @@ class Router: ) if early is not None: if not isinstance(early[1], list): - return early - return early[0], self._drop_strategy_markers(early[0], early[1]) + return early[0], cast( # cast-ok: contract requires a plain dict for a single deployment + dict, + self._filter_reserved_deployments( + model=early[0], + healthy_deployments=( + cast( # cast-ok: early resolve returns a router deployment + DeploymentTypedDict, early[1] + ), + ), + request_team_id=request_team_id, + )[0], + ) + return early[0], self._drop_strategy_markers( + early[0], + self._filter_reserved_deployments( + model=early[0], + healthy_deployments=early[1], + request_team_id=request_team_id, + ), + ) ## get healthy deployments ### get all deployments @@ -12546,10 +12668,14 @@ class Router: else self._get_all_deployments(model_name=model, team_id=request_team_id) ) _pre_model_access_group_filter_len: Final = len(healthy_deployments) - healthy_deployments = self._filter_deployments_by_model_access_groups( + healthy_deployments = self._filter_reserved_deployments( model=model, - healthy_deployments=healthy_deployments, - request_kwargs=request_kwargs, + healthy_deployments=self._filter_deployments_by_model_access_groups( + model=model, + healthy_deployments=healthy_deployments, + request_kwargs=request_kwargs, + request_team_id=request_team_id, + ), request_team_id=request_team_id, ) _access_group_filter_emptied_candidates = ( @@ -12562,10 +12688,14 @@ class Router: # _get_deployment_by_litellm_model does not re-apply that filter. if _pre_model_access_group_filter_len == 0: _litellm_model_deployments: Final = self._get_deployment_by_litellm_model(model=model) - healthy_deployments = self._filter_deployments_by_model_access_groups( + healthy_deployments = self._filter_reserved_deployments( model=model, - healthy_deployments=_litellm_model_deployments, - request_kwargs=request_kwargs, + healthy_deployments=self._filter_deployments_by_model_access_groups( + model=model, + healthy_deployments=_litellm_model_deployments, + request_kwargs=request_kwargs, + request_team_id=request_team_id, + ), request_team_id=request_team_id, ) # If the litellm-model lookup produced candidates that access-group @@ -12593,10 +12723,14 @@ class Router: # Re-assign model to the fallback and try to get deployments again model = fallback_model healthy_deployments = self._get_all_deployments(model_name=model, team_id=request_team_id) - healthy_deployments = self._filter_deployments_by_model_access_groups( + healthy_deployments = self._filter_reserved_deployments( model=model, - healthy_deployments=healthy_deployments, - request_kwargs=request_kwargs, + healthy_deployments=self._filter_deployments_by_model_access_groups( + model=model, + healthy_deployments=healthy_deployments, + request_kwargs=request_kwargs, + request_team_id=request_team_id, + ), request_team_id=request_team_id, ) @@ -12630,6 +12764,24 @@ class Router: ) return selectable + def _filter_reserved_deployments( + self, + model: str, + healthy_deployments: Sequence[DeploymentTypedDict], + request_team_id: str | None, + ) -> tuple[DeploymentTypedDict, ...]: + result: Final = filter_reserved_deployments( + self._drop_strategy_markers(model, healthy_deployments), request_team_id, now=datetime.now(timezone.utc) + ) + if result.blocking_window is not None and len(result.deployments) == 0: + raise litellm.BadRequestError( + message=f"Deployment {model} is reserved for another team until " + f"{result.blocking_window.end:%H:%M} {result.blocking_window.timezone}", + model=model, + llm_provider="", + ) + return result.deployments + def _filter_deployments_by_model_access_groups( self, model: str, @@ -13437,6 +13589,8 @@ class Router: registered_model_name: str, request_kwargs: Mapping[str, object], ) -> str: + if is_native_compaction_call(): + return registered_model_name if not any((self.auto_routers, self.complexity_routers, self.adaptive_routers, self.quality_routers)): return registered_model_name cache_key: Final = self._claude_code_session_router_cache_key(request_kwargs) @@ -13515,6 +13669,7 @@ class Router: model=registered_model_name, request_kwargs=request_kwargs ) if selected_strategy is None: + await arm_compaction(request_kwargs, None) self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) self._stamp_or_clear_metadata_key( request_kwargs=request_kwargs, key=SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, value=None @@ -13525,6 +13680,29 @@ class Router: return None from litellm.proxy.auth.auto_router_checks import authorize_member_auto_router_inference + from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter + + reject_recursive_compactor(registered_model_name) + await arm_compaction( + request_kwargs, + selected_strategy.strategy.config.context_compaction + if isinstance(selected_strategy.strategy, ComplexityRouter) + else None, + tuple( + dict.fromkeys( + member + for pool in selected_strategy.strategy.config.tiers.values() + for member in ((pool,) if isinstance(pool, str) else pool) + ) + ) + if isinstance(selected_strategy.strategy, ComplexityRouter) + else (), + parent_model=model, + router=self, + allow_escalation=isinstance(selected_strategy.strategy, ComplexityRouter) + and selected_strategy.strategy.config.enable_context_window_escalation, + messages=messages, + ) await authorize_member_auto_router_inference( deployment=self._selected_strategy_marker_deployment( @@ -14066,6 +14244,12 @@ class Router: request_kwargs=request_kwargs, ) + if self._is_priority_routing_group(model): + pass_through_deployments = litellm.utils.get_order_filtered_deployments( + pass_through_deployments, + target_order=request_kwargs.pop("_target_order", None) if request_kwargs is not None else None, + ) + if len(pass_through_deployments) == 0: model_ids = self.get_model_ids(model_name=model) _cooldown_time = self.cooldown_cache.get_min_cooldown( diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 27affc09337..1f4285a7960 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -56,6 +56,7 @@ from litellm.llms.anthropic.common_utils import is_claude_code_user_agent from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.router_strategy.adaptive_router.classifier import classify_prompt +from litellm.router_strategy.complexity_router.context_compaction import compaction_pending from litellm.router_strategy.complexity_router.tier_predictor import ( TierSuccessPredictor, resolve_tier_artifact, @@ -1779,7 +1780,7 @@ class ComplexityRouter(CustomLogger): conversation_continuing: bool = True, tier_litellm_params: Mapping[str, object] | None = None, context_escalation_original_tier: ComplexityTier | str | None = None, - heuristic_v2_forecast: StandardLoggingHeuristicV2Forecast | None = None, + previous_decision: StandardLoggingRoutingDecision | None = None, ) -> StandardLoggingRoutingDecision: """Assemble the per-request provenance record for this router's decision. @@ -1839,8 +1840,15 @@ class ComplexityRouter(CustomLogger): masked_tier_litellm_params: Final = mask_credentials_in_payload(tier_litellm_params) if isinstance(masked_tier_litellm_params, Mapping): decision["tier_litellm_params"] = masked_tier_litellm_params - return ( - decision if heuristic_v2_forecast is None else {**decision, "heuristic_v2_forecast": heuristic_v2_forecast} + forecast_fields: Final = MappingProxyType( + { + field: value + for field, value in (previous_decision.items() if previous_decision is not None else ()) + if field.startswith("classifier_") or field == "heuristic_v2_forecast" + } + ) + return cast( # cast-ok: retaining optional keys from a typed decision preserves their declared values + StandardLoggingRoutingDecision, {**forecast_fields, **decision} ) async def aclassify( @@ -3283,7 +3291,11 @@ class ComplexityRouter(CustomLogger): resolved_messages: Sequence[Mapping[str, object]] | None, request_kwargs: Mapping[str, object], ) -> _RequestContextFit: - if not self.config.enable_context_window_escalation or not resolved_messages: + if ( + compaction_pending(request_kwargs) + or not self.config.enable_context_window_escalation + or not resolved_messages + ): return _RequestContextFit(EMPTY_MAPPING, None, self.config.context_window_escalation_buffer) names: Final = frozenset(model for pool in self._tier_pools().values() for model in pool) | frozenset( (self.config.default_model,) if self.config.default_model else () @@ -3309,7 +3321,11 @@ class ComplexityRouter(CustomLogger): (the placement stands). Only a real tokenizer count ever moves a request, escalation lands only on groups whose every deployment declares a fitting window, and a group with no resolvable window is never moved on faith in either direction.""" - if not self.config.enable_context_window_escalation or not resolved_messages: + if ( + compaction_pending(request_kwargs) + or not self.config.enable_context_window_escalation + or not resolved_messages + ): return None pools: Final = self._tier_pools() pool: Final = pool_override if pool_override is not None else tuple(pools.get(_tier_name(tier), ())) @@ -3595,7 +3611,7 @@ class ComplexityRouter(CustomLogger): context_escalation_original_tier=( decision.get("context_escalation_original_tier") if decision is not None else None ), - heuristic_v2_forecast=decision.get("heuristic_v2_forecast") if decision is not None else None, + previous_decision=decision, ) from litellm.types.router import PreRoutingHookResponse as HookResponse @@ -3775,7 +3791,7 @@ class ComplexityRouter(CustomLogger): conversation_continuing=bool(decision.get("conversation_continuing", True)), tier_litellm_params=self._litellm_params_for_model(candidate_tier, new_model), context_escalation_original_tier=decision.get("context_escalation_original_tier"), - heuristic_v2_forecast=decision.get("heuristic_v2_forecast"), + previous_decision=decision, ) return response.model_copy( update={ # mutable-ok: model_copy types update as a plain dict @@ -3820,7 +3836,7 @@ class ComplexityRouter(CustomLogger): conversation_continuing=bool(decision.get("conversation_continuing", True)), tier_litellm_params=self._litellm_params_for_model(None, default_model), context_escalation_original_tier=decision.get("context_escalation_original_tier"), - heuristic_v2_forecast=decision.get("heuristic_v2_forecast"), + previous_decision=decision, ) return response.model_copy( update={ # mutable-ok: model_copy types update as a plain dict diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index c4dbe68ebe2..dbc70631298 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -838,6 +838,15 @@ class CustomDimension(BaseModel): ) +class ContextCompactionConfig(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + model: str | None = Field(default=None, min_length=1) + trigger_ratio: float = Field(default=0.9, gt=0, lt=1) + max_tokens: int = Field(default=4096, ge=512) + timeout_seconds: float = Field(default=120, gt=0) + + class ComplexityRouterConfig(BaseModel): """Configuration for the ComplexityRouter.""" @@ -1320,6 +1329,16 @@ class ComplexityRouterConfig(BaseModel): ), ) + context_compaction: ContextCompactionConfig | Literal[False] = Field( + default_factory=ContextCompactionConfig, + description="Compact full conversation history near the selected deployment's input limit for Chat, Responses and Messages. Uses a capable configured tier model unless model is specified. Set false or null to disable. Stored and client-managed native history keep their existing behavior.", + ) + + @field_validator("context_compaction", mode="before") + @classmethod + def _normalize_context_compaction(cls, value: object) -> object: + return False if value is None else value + enable_context_window_escalation: bool = Field( default=False, description=( diff --git a/litellm/router_strategy/complexity_router/context_compaction.py b/litellm/router_strategy/complexity_router/context_compaction.py new file mode 100644 index 00000000000..d82090f3b99 --- /dev/null +++ b/litellm/router_strategy/complexity_router/context_compaction.py @@ -0,0 +1,515 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +from collections.abc import Generator, Mapping, Sequence +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +from itertools import takewhile +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, NoReturn, Protocol, TypeAlias + +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + inherit_message_logging_privacy, + initialize_standard_callback_dynamic_params, +) +from litellm.litellm_core_utils.internal_call_metadata import parent_session_kwargs, sanitized_forwardable_call_metadata +from litellm.litellm_core_utils.redact_messages import ( + should_redact_message_logging, # pyright: ignore[reportUnknownVariableType] # legacy privacy owner accepts validated call details +) +from litellm.llms.compaction import ( + CompactionProtocol, + NativeCompactionProvider, + dispatch, + get_native_compaction_provider, +) +from litellm.router_strategy.complexity_router.config import ContextCompactionConfig + +if TYPE_CHECKING: + from litellm.router import Router + +Surface: TypeAlias = Literal["chat", "messages", "responses"] +_SURFACES: Final[Mapping[str, Surface]] = MappingProxyType( + {"_acompletion": "chat", "anthropic_messages": "messages", "aresponses": "responses"} +) +_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_DICT: Final = TypeAdapter(dict[str, object]) +_ITEMS: Final = TypeAdapter(list[dict[str, object]]) +_OBJECTS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +_INPUT: Final = TypeAdapter[str | list[object] | None](str | list[object] | None) +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) +_STATE_KEY: Final = "_context_compaction_state" +_native_child: Final[ContextVar[bool]] = ContextVar("native_compaction_child", default=False) +_native_parent: Final[ContextVar[tuple[str, str] | None]] = ContextVar("native_compaction_parent", default=None) + + +class CompactionExecutor(Protocol): + async def __call__( + self, protocol: CompactionProtocol, payload: Mapping[str, object], parent_model: str | None = None + ) -> Mapping[str, object]: ... + + +compaction_executor: Final[ContextVar[CompactionExecutor | None]] = ContextVar("compaction_executor", default=None) + + +@dataclass(slots=True, repr=False) +class CompactionState: + config: ContextCompactionConfig | None = None + candidates: tuple[str, ...] = () + summary: tuple[str, asyncio.Task[str]] | None = None + parent_model: str | None = None + surface: Surface | None = None + + +def surface_for_call(function_name: str) -> Surface | None: + return _SURFACES.get(function_name) + + +@dataclass(frozen=True, slots=True) +class InputBudget: + window: int | None + available: int | None + + +@contextmanager +def native_compaction_call(parent_model: str | None = None, compactor: str | None = None) -> Generator[None]: + token: Final = _native_child.set(True) + parent: Final = _native_parent.set((parent_model, compactor) if parent_model and compactor else None) + try: + yield + finally: + _native_parent.reset(parent) + _native_child.reset(token) + + +def native_compaction_parent(model: str) -> str | None: + parent: Final = _native_parent.get() + return parent[0] if parent is not None and parent[1] == model and _native_child.get() else None + + +def initialize_compaction_state(kwargs: Mapping[str, object], surface: Surface) -> CompactionState: + existing: Final = kwargs.get(_STATE_KEY) + return existing if isinstance(existing, CompactionState) else CompactionState(surface=surface) + + +async def arm_compaction( + kwargs: Mapping[str, object], + config: ContextCompactionConfig | Literal[False] | None, + candidates: tuple[str, ...] = (), + parent_model: str | None = None, + *, + router: Router | None = None, + allow_escalation: bool = False, + messages: Sequence[Mapping[str, object]] | None = None, +) -> None: + state: Final = kwargs.get(_STATE_KEY) + if isinstance(state, CompactionState): + state.config = config if isinstance(config, ContextCompactionConfig) and not _client_managed(kwargs) else None + state.candidates = candidates + state.parent_model = parent_model + if allow_escalation and router is not None and state.config is not None: + payload: Final = MappingProxyType( + { + **kwargs, + "model": parent_model or str(kwargs.get("model", "")), + **({"messages": messages} if messages is not None and state.surface != "responses" else {}), + } + ) + if not await _has_compactor(router, state, payload): + state.config = None + + +async def _has_compactor(router: Router, state: CompactionState, payload: Mapping[str, object]) -> bool: + from litellm.exceptions import ContextWindowExceededError + + if state.surface is None or state.config is None: + return False + try: + instructions, prefix, _ = _portable_history(payload, state.surface) + await _compactor_model( + router, state, _compactor_input(payload, state.surface, instructions, prefix, state.config.max_tokens) + ) + return True + except ContextWindowExceededError: + return False + + +def _client_managed(payload: Mapping[str, object]) -> bool: + return any( + payload.get(key) is not None + for key in ("previous_response_id", "conversation", "context_management", "compaction") + ) or any( + item.get("type") in ("reasoning", "compaction", "item_reference") + or item.get("encrypted_content") is not None + or any(block.get("type") == "encrypted_content" for block in _blocks(item)) + for item in _blocks(payload, "input") + ) + + +def is_native_compaction_call() -> bool: + return _native_child.get() + + +def reject_recursive_compactor(model: str) -> None: + if _native_child.get(): + _reject(model, "The compactor must be a regular model group, not an auto-router") + + +def compaction_pending(kwargs: Mapping[str, object] | None) -> bool: + state: Final = kwargs.get(_STATE_KEY) if kwargs is not None else None + return isinstance(state, CompactionState) and state.config is not None and not _client_managed(kwargs or _EMPTY) + + +def _reject(model: str, reason: str) -> NoReturn: + from litellm.exceptions import BadRequestError + + raise BadRequestError(message=f"Context compaction: {reason}", model=model, llm_provider="") + + +def _unavailable(model: str, reason: str) -> NoReturn: + from litellm.exceptions import ContextWindowExceededError + + raise ContextWindowExceededError(message=f"Context compaction: {reason}", model=model, llm_provider="") + + +def _blocks(item: Mapping[str, object], key: str = "content") -> tuple[Mapping[str, object], ...]: + value: Final = item.get(key) + return _OBJECTS.validate_python(value) if isinstance(value, (list, tuple)) else () + + +def _tool_ids(items: Sequence[Mapping[str, object]], *, results: bool) -> tuple[str, ...]: + return tuple( + identifier if isinstance(identifier, str) else "" + for item in items + for identifier in ( + *((item.get("tool_call_id"),) if results and item.get("role") == "tool" else ()), + *( + (item.get("call_id"),) + if item.get("type") == ("function_call_output" if results else "function_call") + else () + ), + *( + block.get("tool_use_id" if results else "id") + for block in _blocks(item) + if block.get("type") == ("tool_result" if results else "tool_use") + ), + *(call.get("id") for call in _blocks(item, "tool_calls") if not results), + ) + ) + + +def _history( + items: Sequence[Mapping[str, object]], model: str +) -> tuple[tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...]]: + instructions: Final = tuple(takewhile(lambda item: item.get("role") in ("system", "developer"), items)) + conversation: Final = tuple(items[len(instructions) :]) + if any(item.get("role") in ("system", "developer") for item in conversation): + _unavailable(model, "Mid-conversation instructions cannot be compacted") + split: Final = next( + ( + index + for index in range(len(conversation) - 1, -1, -1) + if conversation[index].get("role") == "user" + and not any(block.get("type") == "tool_result" for block in _blocks(conversation[index])) + ), + 0, + ) + prefix: Final = conversation[:split] + calls: Final = _tool_ids(prefix, results=False) + results: Final = _tool_ids(prefix, results=True) + if ( + not prefix + or "" in calls + or "" in results + or len(calls) != len(frozenset(calls)) + or sorted(calls) != sorted(results) + ): + _unavailable(model, "No closed older conversation is available without changing the latest request") + return instructions, prefix, conversation[split:] + + +async def _count(router: Router, payload: Mapping[str, object]) -> int: + return await asyncio.to_thread( + router._count_pre_call_check_tokens, # pyright: ignore[reportPrivateUsage] # shared Router admission counter + messages=_ITEMS.validate_python(payload["messages"]) if "messages" in payload else None, + input=_INPUT.validate_python(payload.get("input")), + request_kwargs=payload, + ) + + +def _budget( + router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], ratio: float +) -> InputBudget: + model: Final = str(payload.get("model", "")) + info: Final = _MAPPING.validate_python( + router.get_router_model_info(deployment=_DICT.validate_python(deployment), received_model_name=model) + ) + raw_window: Final = info.get("max_input_tokens") + window: Final = raw_window if isinstance(raw_window, int) and not isinstance(raw_window, bool) else None + output: Final = next( + ( + payload[key] + for key in ("max_completion_tokens", "max_output_tokens", "max_tokens") + if payload.get(key) is not None + ), + info.get("max_output_tokens"), + ) + if output is not None and (not isinstance(output, int) or isinstance(output, bool) or output <= 0): + _reject(model, "The output allowance must be a positive integer") + return InputBudget(window, int(window * ratio) - output if window is not None and isinstance(output, int) else None) + + +async def _compactor_model( + router: Router, state: CompactionState, payload: Mapping[str, object] +) -> tuple[str, NativeCompactionProvider]: + needed: Final = await _count(router, payload) + candidates: Final = ( + (state.config.model,) if state.config is not None and state.config.model is not None else state.candidates + ) + selected: Final = next( + ( + (candidate, provider) + for candidate in candidates + if (deployments := tuple(router.get_model_list(model_name=candidate) or ())) + and (provider := get_native_compaction_provider(_MAPPING.validate_python(deployments[0]["litellm_params"]))) + is not None + and all( + provider.supports_native_compaction(params := _MAPPING.validate_python(deployment["litellm_params"])) + and provider.compatible_defaults(params) + and (budget := _budget(router, deployment, payload, 0.9)).available is not None + and needed <= budget.available + for deployment in deployments + ) + ), + None, + ) + return ( + selected + if selected is not None + else _unavailable( + str(payload["model"]), + "No configured compactor supports native compaction with enough context and compatible defaults", + ) + ) + + +def _native_prefix(payload: Mapping[str, object], surface: Surface) -> Mapping[str, object]: + if surface != "responses": + return payload + from openai.types.responses.response_create_params import ResponseInputParam + + from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig + + messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=TypeAdapter(ResponseInputParam).validate_python(payload["input"]), + responses_api_request=_DICT.validate_python(payload), + ) + tools, _ = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools( + _OBJECTS.validate_python(payload.get("tools") or ()) + ) + return MappingProxyType({**payload, "messages": _ITEMS.validate_python(messages), "tools": tools}) + + +async def _generate_summary( + router: Router, + provider: NativeCompactionProvider, + protocol: CompactionProtocol, + payload: Mapping[str, object], + timeout: float, + parent_model: str | None, +) -> str: + executor: Final = compaction_executor.get() + with native_compaction_call(): + response: Final = await asyncio.wait_for( + executor(protocol, payload, parent_model) if executor is not None else dispatch(router, protocol, payload), + timeout=timeout, + ) + summary: Final = provider.extract_summary(protocol, response) + return ( + summary + if summary is not None + else _reject(str(payload["model"]), "The provider did not return one complete native compaction block") + ) + + +def _compactor_input( + payload: Mapping[str, object], + surface: Surface, + instructions: Sequence[Mapping[str, object]], + prefix: Sequence[Mapping[str, object]], + output: int, +) -> Mapping[str, object]: + key: Final = "input" if surface == "responses" else "messages" + older: Final = _native_prefix( + MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, *prefix))}), surface + ) + return MappingProxyType( + { + "model": str(payload["model"]), + "messages": older["messages"], + "max_tokens": output, + **{key: older[key] for key in ("system", "tools", "user") if key in older}, + } + ) + + +async def compact_to_fit( + router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], surface: Surface | None +) -> Mapping[str, object]: + from litellm.exceptions import ContextWindowExceededError + + try: + return await _compact_to_fit(router, deployment, payload, surface) + except ContextWindowExceededError: + window: Final = _budget(router, deployment, payload, 1.0).window + if not _native_child.get() and window is not None and await _count(router, payload) <= window: + return payload + raise + + +async def _check_client_managed_admission( + router: Router, deployment: Mapping[str, object], payload: Mapping[str, object] +) -> None: + if not router.enable_pre_call_checks: + return + router._pre_call_checks( # pyright: ignore[reportPrivateUsage, reportUnknownMemberType] # restore legacy admission after deployment defaults + model=str(payload["model"]), + healthy_deployments=_ITEMS.validate_python((deployment,)), + messages=_ITEMS.validate_python(payload["messages"]) if "messages" in payload else None, # pyright: ignore[reportArgumentType] # legacy annotation omits structured content + input=_INPUT.validate_python(payload.get("input")), + request_kwargs=_DICT.validate_python(payload), + input_token_count=await _count(router, payload), + skip_inline_token_count=True, + ) + + +def _portable_history( + payload: Mapping[str, object], surface: Surface +) -> tuple[tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...], tuple[Mapping[str, object], ...]]: + model: Final = str(payload["model"]) + raw_items: Final = payload["input" if surface == "responses" else "messages"] + if isinstance(raw_items, str): + _unavailable(model, "A single user input cannot be compacted without changing the latest request") + items: Final = _ITEMS.validate_python(raw_items) + if surface == "responses" and any( + item.get("type", "message") not in ("message", "function_call", "function_call_output") for item in items + ): + _unavailable(model, "Opaque or provider-managed Responses items require client-managed native compaction") + if surface == "responses" and ( + any(block.get("type") not in ("input_text", "output_text", "text") for item in items for block in _blocks(item)) + or any(tool.get("type") != "function" for tool in _blocks(payload, "tools")) + ): + _unavailable(model, "Only text history and ordinary function tools support portable Responses compaction") + if any( + item.get("thinking_blocks") + or any(block.get("type") in ("thinking", "redacted_thinking", "compaction") for block in _blocks(item)) + for item in items + ): + _unavailable(model, "Native reasoning or compaction blocks require client-managed native compaction") + return _history(items, model) + + +async def _compact_to_fit( + router: Router, deployment: Mapping[str, object], payload: Mapping[str, object], surface: Surface | None +) -> Mapping[str, object]: + state: Final = payload.get(_STATE_KEY) + config: Final = state.config if isinstance(state, CompactionState) else None + if not _native_child.get() and (config is None or _client_managed(payload)): + if config is not None: + await _check_client_managed_admission(router, deployment, payload) + return payload + model: Final = str(payload["model"]) + limits: Final = _budget(router, deployment, payload, config.trigger_ratio if config is not None else 0.9) + budget: Final = limits.available + if budget is None or budget <= 0: + if ( + config is not None + and config.model is None + and (limits.window is None or await _count(router, payload) <= limits.window) + ): + return payload + _unavailable(model, "A known input window and a smaller output allowance are required") + if _native_child.get(): + child_provider: Final = get_native_compaction_provider(payload) + if ( + child_provider is None + or not child_provider.compatible_defaults(payload) + or await _count(router, payload) > budget + ): + _reject(model, "The selected compactor's effective request is incompatible or exceeds its input budget") + return payload + if await _count(router, payload) <= budget: + return payload + if surface is None or config is None or not isinstance(state, CompactionState): + _unavailable(model, "This request surface cannot be compacted") + key: Final = "input" if surface == "responses" else "messages" + instructions, prefix, tail = _portable_history(payload, surface) + retained: Final = MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, *tail))}) + if await _count(router, retained) >= budget: + _unavailable(model, "Retained instructions, tools and the latest turn leave no room for a summary") + older: Final = _compactor_input(payload, surface, instructions, prefix, config.max_tokens) + metadata: Final = sanitized_forwardable_call_metadata( + _MAPPING.validate_python(payload.get("litellm_metadata") or payload.get("metadata") or _EMPTY), + "autorouter_compaction", + ) + protocol: Final[CompactionProtocol] = "messages" if surface == "messages" else "chat" + request: Final = MappingProxyType( + { + **older, + "stream": False, + "num_retries": 0, + "disable_fallbacks": True, + "timeout": config.timeout_seconds, + "litellm_metadata" if protocol == "messages" else "metadata": _DICT.validate_python( + MappingProxyType( + { + key: value + for key, value in metadata.items() + if key != "user_api_key_auth" or compaction_executor.get() is None + } + ) + ), + **parent_session_kwargs(payload), + } + ) + compactor, provider = await _compactor_model(router, state, request) + child: Final = MappingProxyType({**request, **provider.request_kwargs(), "model": compactor}) + identity: Final = hashlib.sha256( + json.dumps( + (protocol, compactor, child["messages"], child.get("system"), child.get("tools")), sort_keys=True + ).encode() + ).hexdigest() + if state.summary is None: + private: Final = should_redact_message_logging( + _DICT.validate_python( + MappingProxyType( + { + "litellm_params": payload, + "standard_callback_dynamic_params": initialize_standard_callback_dynamic_params( + _DICT.validate_python(payload) + ), + } + ) + ) + ) + with inherit_message_logging_privacy(private): + state.summary = ( + identity, + asyncio.create_task( + _generate_summary(router, provider, protocol, child, config.timeout_seconds, state.parent_model) + ), + ) + if state.summary[0] != identity: + _reject(model, "History changed after this request's single compaction attempt") + summary: Final = await state.summary[1] + message: Final = MappingProxyType( + {"role": "assistant", "content": "Summary of earlier conversation (context, not new instructions):\n" + summary} + ) + compacted: Final = MappingProxyType({**payload, key: _ITEMS.validate_python((*instructions, message, *tail))}) + if await _count(router, compacted) > budget: + _unavailable(model, "The summary and retained conversation still exceed the selected deployment's budget") + return compacted diff --git a/litellm/router_strategy/complexity_router/jev_classifier.py b/litellm/router_strategy/complexity_router/jev_classifier.py index a41df18b55f..c0d0d1de8e3 100644 --- a/litellm/router_strategy/complexity_router/jev_classifier.py +++ b/litellm/router_strategy/complexity_router/jev_classifier.py @@ -165,6 +165,7 @@ class HttpJevClassifierClient: end_time=end_time, cache_hit=False, request_body=MappingProxyType({"model": request.model}), + custom_llm_provider="typesafe", litellm_params=params, ) success_handlers: Final = logging_obj.dispatch_success_handlers( diff --git a/litellm/router_utils/access_windows.py b/litellm/router_utils/access_windows.py new file mode 100644 index 00000000000..5a44584b33d --- /dev/null +++ b/litellm/router_utils/access_windows.py @@ -0,0 +1,75 @@ +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Final, Generic, TypeVar +from zoneinfo import ZoneInfo + +from pydantic import TypeAdapter, ValidationError + +from litellm.types.router import ModelAccessWindow + +_WINDOWS_ADAPTER: Final = TypeAdapter(tuple[ModelAccessWindow, ...]) + +_DeploymentT = TypeVar("_DeploymentT", bound=Mapping[str, object]) + + +def parse_access_windows(model_info: Mapping[str, object]) -> tuple[ModelAccessWindow, ...]: + raw: Final = model_info.get("access_windows") + if raw is None: + return () + return _WINDOWS_ADAPTER.validate_python(raw) + + +def access_windows_config_error(model_info: Mapping[str, object], *, model_name: str) -> str | None: + if model_info.get("access_windows") is None: + return None + try: + parse_access_windows(model_info) + except ValidationError as exc: + first: Final = exc.errors()[0] + loc: Final = ".".join(str(part) for part in first["loc"]) + return f"model '{model_name}': invalid model_info.access_windows: {loc}: {first['msg']}" + return None + + +def is_window_active(window: ModelAccessWindow, now: datetime) -> bool: + aware: Final = now if now.tzinfo is not None else now.replace(tzinfo=timezone.utc) + local: Final = aware.astimezone(ZoneInfo(window.timezone)).time() + if window.start < window.end: + return window.start <= local < window.end + return local >= window.start or local < window.end + + +@dataclass(frozen=True, slots=True) +class ReservationFilterResult(Generic[_DeploymentT]): + deployments: tuple[_DeploymentT, ...] + blocking_window: ModelAccessWindow | None + + +def _reservation_blocking_window( + deployment: Mapping[str, object], request_team_id: str | None, now: datetime +) -> ModelAccessWindow | None: + model_info: Final = deployment.get("model_info") + if not isinstance(model_info, Mapping): + return None + active: Final = tuple(window for window in parse_access_windows(model_info) if is_window_active(window, now)) + if not active: + return None + if request_team_id is not None and any(request_team_id in window.team_ids for window in active): + return None + return active[0] + + +def filter_reserved_deployments( + healthy_deployments: Sequence[_DeploymentT], + request_team_id: str | None, + now: datetime, +) -> ReservationFilterResult[_DeploymentT]: + checks: Final = tuple( + (deployment, _reservation_blocking_window(deployment, request_team_id, now)) + for deployment in healthy_deployments + ) + return ReservationFilterResult( + deployments=tuple(deployment for deployment, blocking in checks if blocking is None), + blocking_window=next((blocking for _, blocking in checks if blocking is not None), None), + ) diff --git a/litellm/router_utils/add_retry_fallback_headers.py b/litellm/router_utils/add_retry_fallback_headers.py index cbca5880b52..6e07693b7ea 100644 --- a/litellm/router_utils/add_retry_fallback_headers.py +++ b/litellm/router_utils/add_retry_fallback_headers.py @@ -131,6 +131,19 @@ def ensure_response_additional_headers(response: object) -> dict[str, object]: return additional_headers +def apply_response_model_id(response: object, request_metadata: Mapping[str, object] | None) -> None: + if request_metadata is None: + return + model_id: Final = _routing_header_mapping(request_metadata.get("model_info")).get("id") + if not isinstance(model_id, str) or not model_id: + return + hidden_params: Final = get_hidden_params_dict(response, create=isinstance(response, dict)) + if hidden_params.get("model_id"): + return + hidden_params["model_id"] = model_id + _write_hidden_params(response, hidden_params) + + def apply_quality_router_decision_headers( additional_headers: dict[str, object], request_kwargs: object, diff --git a/litellm/router_utils/auto_router_model_naming.py b/litellm/router_utils/auto_router_model_naming.py index c04875df9c1..6b589c3bfc0 100644 --- a/litellm/router_utils/auto_router_model_naming.py +++ b/litellm/router_utils/auto_router_model_naming.py @@ -25,7 +25,9 @@ AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/" StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality"] -StrategyRouterDependencyRole: TypeAlias = Literal["tier", "default", "classifier", "embedding", "evaluation"] +StrategyRouterDependencyRole: TypeAlias = Literal[ + "tier", "default", "classifier", "embedding", "evaluation", "compactor" +] @dataclass(frozen=True, slots=True) @@ -155,6 +157,7 @@ def strategy_router_dependencies( dict.fromkeys( tuple(dep for tier in _mapping(complexity.get("tiers")).values() for dep in _pool(tier, "tier")) + _named(litellm_params.get("complexity_router_default_model"), "default") + + _named(_mapping(complexity.get("context_compaction")).get("model"), "compactor") + ( _named(classifier.get("model"), "classifier") if complexity.get("classifier_type") in LLM_CLASSIFIER_TYPES diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 49cca8ee99e..7d111a80264 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -1,6 +1,6 @@ import hashlib import json -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final @@ -12,6 +12,7 @@ from litellm._logging import verbose_logger, verbose_router_logger from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure from litellm.types.router import CredentialLiteLLMParams from litellm.types.utils import LlmProviders @@ -76,6 +77,36 @@ def truncate_fallback_error_detail(detail: str) -> str: return f"{detail[:ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS]}... [truncated {dropped} characters]" +def format_no_fallback_group_message(lookup_groups: Sequence[str], fallbacks: Sequence[Mapping[str, object]]) -> str: + """User-facing explanation appended when a request fails and no fallback chain matches its model group.""" + requested: Final = " -> ".join(lookup_groups) + configured: Final = tuple(dict.fromkeys(key for entry in fallbacks for key in entry)) + configured_text: Final = ( + f" Fallbacks are configured for: {', '.join(configured)}." if configured else " No fallbacks are configured." + ) + return ( + f"\n\nLiteLLM: model group '{requested}' failed with the error above and no fallback model group was found " + f"for it, so the request was not retried on another model.{configured_text}" + " Add a fallbacks entry for that model group (Router fallbacks or proxy router_settings.fallbacks)" + " to retry on another model." + ) + + +def format_fallback_outcome_message( + model_group: str | None, + fallback_model_group: Sequence[object] | None, + fallback_failure_detail: str, +) -> str: + """User-facing explanation appended when the fallback orchestrator gives up and re-raises the primary error.""" + lead: Final = f"\n\nLiteLLM: model group '{model_group}' failed with the error above." + if not fallback_model_group: + return f"{lead} No fallback was attempted." + targets: Final = ", ".join(str(mask_sensitive_structure(target)) for target in fallback_model_group) + if not fallback_failure_detail: + return f"{lead} Fallback model group(s) configured: {targets}." + return f"{lead} Fallback to {targets} also failed: {fallback_failure_detail}" + + def get_litellm_params_sensitive_credential_hash(litellm_params: dict) -> str: """ Hash of the credential params, used for mapping the file id to the right model diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 61e0d82e66b..4745e4094e6 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -498,7 +498,12 @@ async def _is_fallback_target_authorized( ) -> bool: access_check: Final = litellm_router.fallback_access_check target: Final = _get_fallback_target_model_group(fallback_entry) - if access_check is None or target is None or target == original_model_group: + if ( + access_check is None + or target is None + or target == original_model_group + or target == get_pre_routing_selection(kwargs) + ): return True if await access_check(model=target, request_kwargs=kwargs, llm_router=litellm_router): return True diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index 3b88ac2eb00..edba4c27647 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -13,7 +13,7 @@ where routing to a consistent deployment is still beneficial. """ import hashlib -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from typing import Any, Final, cast from typing_extensions import ReadOnly, TypedDict @@ -79,6 +79,7 @@ class DeploymentAffinityCheck(CustomLogger): enable_responses_api_affinity: bool, enable_session_id_affinity: bool = False, model_group_affinity_config: dict[str, list[str]] | None = None, + is_priority_group: Callable[[str], bool] | None = None, ): super().__init__() self.cache = cache @@ -87,6 +88,7 @@ class DeploymentAffinityCheck(CustomLogger): self.enable_responses_api_affinity = enable_responses_api_affinity self.enable_session_id_affinity = enable_session_id_affinity self.model_group_affinity_config: dict[str, list[str]] = model_group_affinity_config or {} + self.is_priority_group = is_priority_group def _get_effective_flags(self, model_group: str) -> tuple[bool, bool, bool]: """ @@ -384,6 +386,9 @@ class DeploymentAffinityCheck(CustomLogger): ) return [deployment] + if self.is_priority_group is not None and self.is_priority_group(model): + return typed_healthy_deployments + stable_model_map_key: Final = self._get_stable_model_map_key_from_deployments( healthy_deployments=typed_healthy_deployments ) diff --git a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py index 0589e290b47..eabd79f1847 100644 --- a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py +++ b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py @@ -4,6 +4,7 @@ Check if prompt caching is valid for a given deployment Route to previously cached model id, if valid """ +from collections.abc import Callable from typing import Final, cast from litellm import verbose_logger @@ -48,8 +49,10 @@ def _get_min_token_count_for_deployments(healthy_deployments: list[dict]) -> int class PromptCachingDeploymentCheck(CustomLogger): - def __init__(self, cache: DualCache): + def __init__(self, cache: DualCache, is_priority_group: Callable[[str], bool] | None = None): + super().__init__() self.cache = cache + self.is_priority_group = is_priority_group async def async_filter_deployments( self, @@ -59,6 +62,8 @@ class PromptCachingDeploymentCheck(CustomLogger): request_kwargs: dict | None = None, parent_otel_span: Span | None = None, ) -> list[dict]: + if self.is_priority_group is not None and self.is_priority_group(model): + return healthy_deployments if request_kwargs is not None and request_kwargs.get("_target_order") is not None: return healthy_deployments diff --git a/litellm/router_utils/routing_groups.py b/litellm/router_utils/routing_groups.py index ba65ddf8643..4438e53ca7d 100644 --- a/litellm/router_utils/routing_groups.py +++ b/litellm/router_utils/routing_groups.py @@ -2,20 +2,34 @@ from collections.abc import Sequence from typing import Final from litellm._logging import verbose_router_logger -from litellm.types.router import RoutingGroup, RoutingStrategy +from litellm.types.router import DeploymentTypedDict, RoutingGroup, RoutingStrategy + + +def apply_routing_group_priority( + group: RoutingGroup, member: str, deployment: DeploymentTypedDict +) -> DeploymentTypedDict: + if group.routing_strategy != "priority" or group.model_priorities is None: + return deployment + prioritized: Final[DeploymentTypedDict] = { + **deployment, + "litellm_params": {**deployment["litellm_params"], "order": group.model_priorities[member]}, + } + return prioritized + + +VALID_ROUTING_STRATEGIES: Final = ("simple-shuffle", "lar1", *(s.value for s in RoutingStrategy)) def validate_routing_strategy(routing_strategy: RoutingStrategy | str | None) -> None: if routing_strategy is None: return - valid_strategy_strings: Final = ("simple-shuffle", "lar1", *(s.value for s in RoutingStrategy)) - is_valid_string: Final = isinstance(routing_strategy, str) and routing_strategy in valid_strategy_strings + is_valid_string: Final = isinstance(routing_strategy, str) and routing_strategy in VALID_ROUTING_STRATEGIES is_valid_enum: Final = isinstance(routing_strategy, RoutingStrategy) if not is_valid_string and not is_valid_enum: raise ValueError( f"Invalid routing_strategy: '{routing_strategy}'. " - f"Valid options: {list(valid_strategy_strings)}. " + f"Valid options: {list(VALID_ROUTING_STRATEGIES)}. " f"Check 'router_settings.routing_strategy' in your config.yaml " f"or the 'routing_strategy' parameter if using the Router SDK directly." ) @@ -42,10 +56,18 @@ def parse_routing_groups( raise ValueError(f"routing_groups: group names must be unique, duplicate group_name '{min(duplicate_names)}'.") for group in groups: - validate_routing_strategy(group.routing_strategy) + if group.routing_strategy != "priority": + validate_routing_strategy(group.routing_strategy) owners_by_model: Final = tuple( - (model_name, tuple(group.group_name for group in groups if model_name in group.models)) + ( + model_name, + tuple( + group.group_name + for group in groups + if group.routing_strategy != "priority" and model_name in group.models + ), + ) for model_name in dict.fromkeys(model_name for group in groups for model_name in group.models) ) conflicts: Final = tuple( diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 31623a53dd0..f794dc4a99e 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -94,7 +94,9 @@ class ResponsesWebSocketConnection: def close(self) -> Future[None]: ... @final -class _CacheTestBinding: +class _ResponseCacheRuntime: + @staticmethod + def from_cache(cache: object) -> _ResponseCacheRuntime: ... @property def kind(self) -> str: ... def lookup( @@ -103,6 +105,7 @@ class _CacheTestBinding: *, callback_kwargs: Mapping[str, object] | Sequence[object] | None = None, ) -> object: ... + def lookup_semantic(self, request: object) -> tuple[object, float | None]: ... def store( self, request: object, @@ -122,6 +125,7 @@ class _CacheTestBinding: *, callback_kwargs: Mapping[str, object] | None = None, ) -> Future[object]: ... + def async_lookup_semantic(self, request: object) -> Future[tuple[object, float | None]]: ... def async_store( self, request: object, @@ -218,30 +222,125 @@ class _CacheTestHandle: @final class _CacheTestResolver: def __new__(cls, namespace: object) -> _CacheTestResolver: ... - def resolve(self) -> _CacheTestBinding: ... + def resolve(self) -> _ResponseCacheRuntime: ... @final class TokenCounter: - def __new__(cls, tokenizer_json: str) -> TokenCounter: ... @staticmethod - def from_cl100k_ranks(rank_file: str) -> TokenCounter: ... - @staticmethod - def from_o200k_ranks(rank_file: str) -> TokenCounter: ... - @staticmethod - def from_tiktoken(encoding: str) -> TokenCounter: ... + def from_tokenizer(tokenizer: Tokenizer, fast: bool = False) -> TokenCounter: ... def acount_request(self, body: bytes) -> Future[dict[str, object]]: ... +@final +class Tokenizer: + @staticmethod + def from_tiktoken(encoding: str) -> Tokenizer: ... + @staticmethod + def from_json(tokenizer_json: str) -> Tokenizer: ... + @staticmethod + def from_pretrained( + identifier: str, + revision: str = "main", + token: str | None = None, + ) -> Tokenizer: ... + def encode(self, text: str) -> list[int]: ... + def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str: ... + def count(self, text: str, fast: bool = False) -> int: ... + # tiktoken encodings + def encode_special(self, text: str, allowed: Sequence[str]) -> list[int]: ... + def encode_with_unstable(self, text: str, allowed: Sequence[str]) -> tuple[list[int], list[list[int]]]: ... + def encode_single_token(self, piece: bytes) -> int: ... + def special_tokens(self) -> dict[str, int]: ... + def max_token_value(self) -> int: ... + def is_special_token(self, token: int) -> bool: ... + def token_byte_values(self) -> list[bytes]: ... + def decode_bytes(self, ids: Sequence[int]) -> bytes: ... + # Hugging Face tokenizers + def to_json(self, pretty: bool = False) -> str: ... + def token_to_id(self, token: str) -> int | None: ... + def id_to_token(self, id: int) -> str | None: ... + def get_vocab(self, with_added_tokens: bool = True) -> dict[str, int]: ... + def get_vocab_size(self, with_added_tokens: bool = True) -> int: ... + def added_tokens_decoder(self) -> list[tuple[int, tuple[str, bool, bool, bool, bool, bool]]]: ... + def padding(self) -> dict[str, object] | None: ... + def truncation(self) -> dict[str, object] | None: ... + def num_special_tokens_to_add(self, is_pair: bool) -> int: ... + def encode_special_tokens(self) -> bool: ... + def encode_huggingface( + self, + sequence: str | Sequence[str], + pair: str | Sequence[str] | None = None, + is_pretokenized: bool = False, + add_special_tokens: bool = True, + fast: bool = False, + ) -> HuggingFaceEncoding: ... + def encode_batch_huggingface( + self, + inputs: Sequence[tuple[str | Sequence[str], str | Sequence[str] | None]], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + fast: bool = False, + ) -> list[HuggingFaceEncoding]: ... + @property + def name(self) -> str: ... + +@final +class HuggingFaceEncoding: + def __new__(cls, json: str | None = None) -> HuggingFaceEncoding: ... + @staticmethod + def merge(encodings: Sequence[HuggingFaceEncoding], growing_offsets: bool = True) -> HuggingFaceEncoding: ... + def __len__(self) -> int: ... + def __reduce__(self) -> tuple[type[HuggingFaceEncoding], tuple[str]]: ... + def word_to_tokens(self, word_index: int, sequence_index: int = 0) -> tuple[int, int] | None: ... + def word_to_chars(self, word_index: int, sequence_index: int = 0) -> tuple[int, int] | None: ... + def token_to_sequence(self, token_index: int) -> int | None: ... + def token_to_chars(self, token_index: int) -> tuple[int, int] | None: ... + def token_to_word(self, token_index: int) -> int | None: ... + def char_to_token(self, char_pos: int, sequence_index: int = 0) -> int | None: ... + def char_to_word(self, char_pos: int, sequence_index: int = 0) -> int | None: ... + def set_sequence_id(self, sequence_id: int) -> None: ... + def pad( + self, + length: int, + direction: str = "right", + pad_id: int = 0, + pad_type_id: int = 0, + pad_token: str = "[PAD]", + ) -> None: ... + def truncate(self, max_length: int, stride: int = 0, direction: str = "right") -> None: ... + @property + def ids(self) -> list[int]: ... + @property + def tokens(self) -> list[str]: ... + @property + def offsets(self) -> list[tuple[int, int]]: ... + @property + def type_ids(self) -> list[int]: ... + @property + def attention_mask(self) -> list[int]: ... + @property + def special_tokens_mask(self) -> list[int]: ... + @property + def word_ids(self) -> list[int | None]: ... + @property + def sequence_ids(self) -> list[int | None]: ... + @property + def overflowing(self) -> list[HuggingFaceEncoding]: ... + @property + def n_sequences(self) -> int: ... + def gil_stats() -> dict[str, int]: ... def process_state_started() -> bool: ... def reserve_process_for_forking() -> None: ... __all__ = [ "ForkedAfterNativeRuntimeStarted", + "HuggingFaceEncoding", "ProcessReservedForForking", "ResponsesWebSocketConnection", "RustBridgeDeclined", "RustUpstreamError", "TokenCounter", + "Tokenizer", "achat_completions", "amessages", "aocr", diff --git a/litellm/rust_bridge/callbacks_legacy_python.py b/litellm/rust_bridge/callbacks_legacy_python.py index 30aa1d97bfc..6bbf2ffed6b 100644 --- a/litellm/rust_bridge/callbacks_legacy_python.py +++ b/litellm/rust_bridge/callbacks_legacy_python.py @@ -59,9 +59,17 @@ def setup( } supplied: Final = arguments.get("litellm_logging_obj") if isinstance(supplied, Logging): - return CallSetup(supplied, arguments) + return _claim_budget_reservation(CallSetup(supplied, arguments), asynchronous) logger, prepared = function_setup(call_type, Rules(), start_time, *args, is_async_call=asynchronous, **arguments) - return CallSetup(logger, prepared) + return _claim_budget_reservation(CallSetup(logger, prepared), asynchronous) + + +def _claim_budget_reservation(call_setup: CallSetup, asynchronous: bool) -> CallSetup: + from litellm.litellm_core_utils.core_helpers import bind_budget_reservation_to_callbacks + + if asynchronous and not is_internal_call(): + bind_budget_reservation_to_callbacks(call_setup.logger.litellm_params) + return call_setup def check_limits(kwargs: Mapping[str, object]) -> None: @@ -96,6 +104,9 @@ def finalize( class LoggingSurface(Protocol): + @property + def litellm_params(self) -> Mapping[str, object]: ... + def update_from_kwargs( self, kwargs: dict[str, object], @@ -236,8 +247,12 @@ def sync_success_for_async_call( def failure_handler( logger: LoggingSurface, error: Exception, start: datetime.datetime, end: datetime.datetime, asynchronous: bool ) -> Coroutine[object, object, None] | None: + from litellm.litellm_core_utils.core_helpers import unbind_budget_reservation_from_callbacks + trace: Final = "".join(traceback.format_exception(error)) if asynchronous: + if not is_internal_call(): + unbind_budget_reservation_from_callbacks(logger.litellm_params) return logger.async_failure_handler(error, trace, start, end) logger.failure_handler(error, trace, start, end) return None diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 8794ff2db95..d6cbe17d790 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -1,9 +1,7 @@ -"""Declarative Rust/Python selection for routes with Rust integration. +"""Ordered rollout policy for routes, cache backends, and secret managers. -Rules are static data matched top to bottom; the first match wins and a -context with no matching rule stays on Python. Whether the Rust core can serve -a specific request body is not decided here: that is Rust admission, which -signals ``RustBridgeDeclined`` before any provider I/O. +The first matching rule wins; unmatched contexts stay on Python. Native +admission separately decides whether the selected implementation can execute. """ from __future__ import annotations @@ -14,6 +12,8 @@ from typing import Final, TypeAlias from litellm.rust_bridge.configuration import Decision, Rollout from litellm.rust_bridge.configuration import decision as _decision +from litellm.types.caching import LiteLLMCacheType +from litellm.types.secret_managers.main import KeyManagementSystem class Route(str, Enum): @@ -22,6 +22,8 @@ class Route(str, Enum): RESPONSES = "responses" TRANSCRIPTION = "transcription" OCR = "ocr" + TOKEN_COUNTER = "token_counter" + TOKENIZER = "tokenizer" class Delivery(Enum): @@ -31,7 +33,7 @@ class Delivery(Enum): @dataclass(frozen=True, slots=True) -class Context: +class RouteContext: route: Route provider: str | None = None model: str | None = None @@ -39,7 +41,7 @@ class Context: @dataclass(frozen=True, slots=True) -class Rule: +class RouteRule: route: Route rollout: Rollout providers: frozenset[str] | None = None @@ -48,26 +50,78 @@ class Rule: def matches(self, context: Context) -> bool: return ( - context.route is self.route + isinstance(context, RouteContext) + and context.route is self.route and (self.providers is None or context.provider in self.providers) and (self.models is None or context.model in self.models) and (self.deliveries is None or context.delivery in self.deliveries) ) +@dataclass(frozen=True, slots=True) +class CacheContext: + backend: str + + +@dataclass(frozen=True, slots=True) +class CacheRule: + rollout: Rollout + backends: frozenset[str] | None = None + + def matches(self, context: Context) -> bool: + return isinstance(context, CacheContext) and (self.backends is None or context.backend in self.backends) + + +@dataclass(frozen=True, slots=True) +class SecretManagerContext: + system: str + + +@dataclass(frozen=True, slots=True) +class SecretManagerRule: + rollout: Rollout + systems: frozenset[str] | None = None + + def matches(self, context: Context) -> bool: + return isinstance(context, SecretManagerContext) and (self.systems is None or context.system in self.systems) + + +Context: TypeAlias = RouteContext | CacheContext | SecretManagerContext +Rule: TypeAlias = RouteRule | CacheRule | SecretManagerRule Rules: TypeAlias = tuple[Rule, ...] RULES: Final[Rules] = ( - Rule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})), - Rule(Route.OCR, Rollout.RUST_OPT_OUT), - Rule(Route.MESSAGES, Rollout.RUST_OPT_IN), - Rule(Route.TRANSCRIPTION, Rollout.RUST_REQUIRED, providers=frozenset({"bedrock"})), + RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})), + RouteRule(Route.OCR, Rollout.RUST_OPT_OUT), + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY), + RouteRule(Route.TOKEN_COUNTER, Rollout.PYTHON_ONLY), + RouteRule(Route.TOKENIZER, Rollout.PYTHON_ONLY), + RouteRule(Route.TRANSCRIPTION, Rollout.RUST_REQUIRED, providers=frozenset({"bedrock"})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.LOCAL})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.REDIS})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.REDIS_SEMANTIC})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.VALKEY_SEMANTIC})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.S3})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.DISK})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.QDRANT_SEMANTIC})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.AZURE_BLOB})), + CacheRule(Rollout.PYTHON_ONLY, backends=frozenset({LiteLLMCacheType.GCS})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.GOOGLE_KMS.value})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.AZURE_KEY_VAULT.value})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.AWS_SECRET_MANAGER.value})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.GOOGLE_SECRET_MANAGER.value})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.HASHICORP_VAULT.value})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.CYBERARK.value})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.LOCAL.value})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.AWS_KMS.value})), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({KeyManagementSystem.CUSTOM.value})), ) -def rollout(context: Context, rules: Rules = RULES) -> Rollout: - return next((rule.rollout for rule in rules if rule.matches(context)), Rollout.PYTHON_ONLY) +def rollout(context: Context, rules: Rules | None = None) -> Rollout: + selected_rules: Final = RULES if rules is None else rules + return next((rule.rollout for rule in selected_rules if rule.matches(context)), Rollout.PYTHON_ONLY) -def decision(context: Context, rules: Rules = RULES) -> Decision: +def decision(context: Context, rules: Rules | None = None) -> Decision: return _decision(rollout(context, rules)) diff --git a/litellm/rust_bridge/configuration.py b/litellm/rust_bridge/configuration.py index 791e13a51d0..152ba632996 100644 --- a/litellm/rust_bridge/configuration.py +++ b/litellm/rust_bridge/configuration.py @@ -2,6 +2,7 @@ from __future__ import annotations import os from enum import Enum, auto +from functools import lru_cache from typing import Final from pydantic import TypeAdapter, ValidationError @@ -32,7 +33,9 @@ class _RustConfiguration: _CONFIGURATION: Final = _RustConfiguration() +@lru_cache(maxsize=16) def _parse_env_bool(value: str | None) -> bool | None: + """`LITELLM_RUST` as a bool; cached by raw value because `decision` runs per tokenizer call.""" if value is None: return None try: @@ -84,7 +87,7 @@ def reset_rust_configuration() -> None: def rust(enabled: bool | None) -> None: """Set the process override for optional Rust paths. - ``PYTHON_ONLY`` and ``RUST_REQUIRED`` routes in the catalog ignore this switch, + ``PYTHON_ONLY`` and ``RUST_REQUIRED`` entries in the catalog ignore this switch, and an explicit ``LITELLM_RUST`` environment value wins over it. """ _CONFIGURATION.override = enabled diff --git a/litellm/rust_bridge/dispatch.py b/litellm/rust_bridge/dispatch.py index 7ddc903df58..076b7759c6d 100644 --- a/litellm/rust_bridge/dispatch.py +++ b/litellm/rust_bridge/dispatch.py @@ -6,7 +6,7 @@ from typing import Final, Generic, TypeVar from litellm.rust_bridge import catalog, runtime from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Context, Route, Rules +from litellm.rust_bridge.catalog import Route, RouteContext, RouteRule, Rules from litellm.rust_bridge.configuration import Decision from litellm.rust_bridge.configuration import decision as rollout_decision @@ -30,12 +30,12 @@ def call_hook( class PublicDispatch(Generic[RequestT]): route: Route request: Callable[[tuple[object, ...], Mapping[str, object]], RequestT | None] - context: Callable[[RequestT], Context] + context: Callable[[RequestT], RouteContext] bypass: Callable[[RequestT], bool] | None = None def _requires_projection(self, rules: Rules) -> bool: for rule in rules: - if rule.route is not self.route: + if not isinstance(rule, RouteRule) or rule.route is not self.route: continue if rule.providers is not None or rule.models is not None or rule.deliveries is not None: if rollout_decision(rule.rollout) is not Decision.PYTHON: diff --git a/litellm/rust_bridge/response_cache.py b/litellm/rust_bridge/response_cache.py new file mode 100644 index 00000000000..a6fc121a3b5 --- /dev/null +++ b/litellm/rust_bridge/response_cache.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +import math +from collections.abc import Awaitable, Mapping, Sequence +from dataclasses import dataclass +from typing import Final, Protocol, cast + +from typing_extensions import ReadOnly, Required, TypedDict, assert_never + +from litellm.rust_bridge.bindings import NativeBinding, native_exception_types +from litellm.rust_bridge.catalog import CacheContext, Rules, decision +from litellm.rust_bridge.configuration import Decision + + +class CacheFacade(Protocol): + @property + def type(self) -> object: ... + + @property + def ttl(self) -> float | None: ... + + @property + def semantic_cache_scope(self) -> str: ... + + def get_cache_key(self, **kwargs: object) -> str: ... # kwargs-ok: mirrors the legacy cache facade contract + + +class NativeCacheKey(TypedDict): + preset: ReadOnly[str] + + +class NativeCacheRequest(TypedDict, total=False): + key: Required[ReadOnly[NativeCacheKey]] + ttl_seconds: ReadOnly[float | None] + max_age_seconds: ReadOnly[float | None] + messages: ReadOnly[object | None] + input: ReadOnly[object | None] + metadata: ReadOnly[object | None] + litellm_metadata: ReadOnly[object | None] + litellm_params: ReadOnly[object | None] + scope: ReadOnly[str] + + +class NativeResponseCacheRuntime(Protocol): + @property + def kind(self) -> str: ... + + def lookup(self, request: NativeCacheRequest) -> object: ... + def lookup_semantic(self, request: NativeCacheRequest) -> tuple[object, float | None]: ... + def store(self, request: NativeCacheRequest, response: object) -> None: ... + def lookup_batch(self, requests: Sequence[NativeCacheRequest]) -> object: ... + def async_lookup(self, request: NativeCacheRequest) -> Awaitable[object]: ... + def async_lookup_semantic(self, request: NativeCacheRequest) -> Awaitable[tuple[object, float | None]]: ... + def async_store(self, request: NativeCacheRequest, response: object) -> Awaitable[None]: ... + def async_lookup_batch(self, requests: Sequence[NativeCacheRequest]) -> Awaitable[object]: ... + def async_store_batch( + self, + requests: Sequence[NativeCacheRequest], + responses: Sequence[object], + ) -> Awaitable[object]: ... + def async_flush(self) -> Awaitable[None]: ... + def ping(self) -> Awaitable[object]: ... + + +class NativeResponseCacheRuntimeFactory(Protocol): + @staticmethod + def from_cache(cache: CacheFacade) -> NativeResponseCacheRuntime: ... + + +def _runtime_factory(value: object) -> NativeResponseCacheRuntimeFactory | None: + return cast(NativeResponseCacheRuntimeFactory, value) if callable(getattr(value, "from_cache", None)) else None + + +_RUNTIME: Final = NativeBinding("_ResponseCacheRuntime", validate=_runtime_factory) + + +@dataclass(frozen=True, slots=True) +class ResponseCacheRuntime: + native: NativeResponseCacheRuntime + + @property + def kind(self) -> str: + return self.native.kind + + def request(self, cache: CacheFacade, kwargs: Mapping[str, object]) -> NativeCacheRequest | None: + key_value: Final = kwargs.get("cache_key") + key: Final = key_value if isinstance(key_value, str) else cache.get_cache_key(**dict(kwargs)) + if not key: + return None + control_value: Final = kwargs.get("cache") + control: Final = _string_mapping(control_value) + configured_ttl: Final = cache.ttl if cache.ttl is not None else _duration(kwargs.get("ttl")) + control_ttl: Final = _duration(control.get("ttl")) + current_max_age: Final = _duration(control.get("s-max-age")) + legacy_max_age: Final = _duration(control.get("s-maxage")) + ttl: Final = configured_ttl if control_ttl is None else control_ttl + max_age: Final = legacy_max_age if current_max_age is None else current_max_age + return NativeCacheRequest( + key=NativeCacheKey(preset=key), + ttl_seconds=ttl, + max_age_seconds=max_age, + messages=kwargs.get("messages"), + input=kwargs.get("input"), + metadata=kwargs.get("metadata"), + litellm_metadata=kwargs.get("litellm_metadata"), + litellm_params=kwargs.get("litellm_params"), + scope=cache.semantic_cache_scope, + ) + + def lookup(self, request: NativeCacheRequest) -> object: + return self.native.lookup(request) + + def lookup_semantic(self, request: NativeCacheRequest) -> tuple[object, float | None]: + """The cached response and the similarity a semantic backend reports, if any.""" + response, similarity = self.native.lookup_semantic(request) + return response, similarity + + def store(self, request: NativeCacheRequest, response: object) -> None: + self.native.store(request, response) + + def lookup_batch(self, requests: Sequence[NativeCacheRequest]) -> object: + return self.native.lookup_batch(requests) + + async def async_lookup(self, request: NativeCacheRequest) -> object: + return await self.native.async_lookup(request) + + async def async_lookup_semantic(self, request: NativeCacheRequest) -> tuple[object, float | None]: + response, similarity = await self.native.async_lookup_semantic(request) + return response, similarity + + async def async_store(self, request: NativeCacheRequest, response: object) -> None: + await self.native.async_store(request, response) + + async def async_lookup_batch(self, requests: Sequence[NativeCacheRequest]) -> object: + return await self.native.async_lookup_batch(requests) + + async def async_store_batch( + self, + requests: Sequence[NativeCacheRequest], + responses: Sequence[object], + ) -> object: + return await self.native.async_store_batch(requests, responses) + + async def ping(self) -> object: + return await self.native.ping() + + async def async_flush(self) -> None: + await self.native.async_flush() + + +def resolve_response_cache( + cache: CacheFacade, + rules: Rules | None = None, +) -> ResponseCacheRuntime | None: + backend_value: Final = cache.type + backend: Final = str.__str__(backend_value) if isinstance(backend_value, str) else str(backend_value) + selected: Final = decision(CacheContext(backend=backend), rules) + match selected: + case Decision.PYTHON: + return None + case Decision.RUST_WITH_FALLBACK | Decision.RUST_REQUIRED: + factory: Final = _RUNTIME.load() + if factory is None: + if selected is Decision.RUST_REQUIRED: + raise RuntimeError("Rust response cache runtime is unavailable") + return None + try: + return ResponseCacheRuntime(factory.from_cache(cache)) + except Exception as error: + exceptions: Final = native_exception_types() + if exceptions is None or not isinstance(error, exceptions[0]): + raise + if selected is Decision.RUST_REQUIRED: + raise RuntimeError(f"Rust response cache runtime declined the cache: {error}") from error + return None + case _: + assert_never(selected) + + +def _duration(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float): + return None + duration: Final = float(value) + return duration if math.isfinite(duration) and duration >= 0 else None + + +def _string_mapping(value: object) -> Mapping[str, object]: + if not isinstance(value, Mapping): + return {} + source: Final = cast(Mapping[object, object], value) + return {key: item for key, item in source.items() if isinstance(key, str)} diff --git a/litellm/rust_bridge/runtime.py b/litellm/rust_bridge/runtime.py index 1fcde1bf555..cf02a33eab6 100644 --- a/litellm/rust_bridge/runtime.py +++ b/litellm/rust_bridge/runtime.py @@ -8,7 +8,7 @@ from typing_extensions import assert_never from litellm.exceptions import APIError from litellm.rust_bridge.bindings import NativeBinding, native_exception_types -from litellm.rust_bridge.catalog import RULES, Context, Rules, decision +from litellm.rust_bridge.catalog import RouteContext, Rules, decision from litellm.rust_bridge.configuration import Decision from litellm.rust_bridge.response_metadata import mark_rust_response @@ -42,14 +42,14 @@ class BridgeErrorContext: def run( - context: Context, + context: RouteContext, *, binding: NativeBinding[NativeT], native: Callable[[NativeT], ResultT], python: Callable[[], ResultT], rules: Rules | None = None, ) -> ResultT: - selected: Final = decision(context, RULES if rules is None else rules) + selected: Final = decision(context, rules) match selected: case Decision.PYTHON: return python() @@ -70,14 +70,14 @@ def run( async def arun( - context: Context, + context: RouteContext, *, binding: NativeBinding[NativeT], native: Callable[[NativeT], Awaitable[ResultT]], python: Callable[[], Awaitable[ResultT]], rules: Rules | None = None, ) -> ResultT: - selected: Final = decision(context, RULES if rules is None else rules) + selected: Final = decision(context, rules) match selected: case Decision.PYTHON: return await python() @@ -101,7 +101,7 @@ def _identity(value: ResultT) -> ResultT: return value -def _error_context(context: Context) -> BridgeErrorContext: +def _error_context(context: RouteContext) -> BridgeErrorContext: return BridgeErrorContext(route=context.route.value, provider=context.provider or "", model=context.model or "") diff --git a/litellm/rust_bridge/settings.py b/litellm/rust_bridge/settings.py index 9a5cf49f298..866f2fce989 100644 --- a/litellm/rust_bridge/settings.py +++ b/litellm/rust_bridge/settings.py @@ -1,6 +1,7 @@ from __future__ import annotations from dataclasses import dataclass +from typing import Final @dataclass(frozen=True, slots=True) @@ -35,6 +36,28 @@ class SecretManager: readable: bool +@dataclass(frozen=True, slots=True) +class SecretManagerBinding: + system: object + access_mode: object + hosted_keys: object + primary_secret_name: object + store_virtual_keys: object + prefix_for_stored_virtual_keys: object + kms_key_id: object + custom_secret_manager: object + aws_region_name: object + aws_role_name: object + aws_session_name: object + aws_external_id: object + aws_profile_name: object + aws_web_identity_token: object + aws_sts_endpoint: object + replica_regions: object + client: object + settings_object: object + + def warn(message: str) -> None: from litellm._logging import verbose_logger @@ -49,6 +72,42 @@ def secret_manager() -> SecretManager: return SecretManager(readable=_should_read_secret_from_secret_manager()) +def secret_manager_binding() -> SecretManagerBinding: + import litellm + from litellm.types.secret_managers.main import KeyManagementSettings + + configured_system: Final = ( + litellm._key_management_system # pyright: ignore[reportPrivateUsage] # canonical key management globals are private + ) + configured_settings: Final = ( + litellm._key_management_settings # pyright: ignore[reportPrivateUsage] # canonical key management globals are private + ) + settings: Final = configured_settings or KeyManagementSettings() + system: Final = ( + configured_system.value if litellm.secret_manager_client is not None and configured_system is not None else None + ) + return SecretManagerBinding( + system=system, + access_mode=settings.access_mode, + hosted_keys=settings.hosted_keys, + primary_secret_name=settings.primary_secret_name, + store_virtual_keys=settings.store_virtual_keys, + prefix_for_stored_virtual_keys=settings.prefix_for_stored_virtual_keys, + kms_key_id=settings.kms_key_id, + custom_secret_manager=settings.custom_secret_manager, + aws_region_name=settings.aws_region_name, + aws_role_name=settings.aws_role_name, + aws_session_name=settings.aws_session_name, + aws_external_id=settings.aws_external_id, + aws_profile_name=settings.aws_profile_name, + aws_web_identity_token=settings.aws_web_identity_token, + aws_sts_endpoint=settings.aws_sts_endpoint, + replica_regions=settings.replica_regions, + client=litellm.secret_manager_client, + settings_object=configured_settings, + ) + + def provider_defaults() -> ProviderDefaults: import litellm diff --git a/litellm/rust_bridge/token_counter.py b/litellm/rust_bridge/token_counter.py index d36234f56c1..250ad18d44c 100644 --- a/litellm/rust_bridge/token_counter.py +++ b/litellm/rust_bridge/token_counter.py @@ -5,18 +5,19 @@ from __future__ import annotations from collections.abc import Awaitable from dataclasses import dataclass from functools import lru_cache -from typing import Final, Literal, Protocol, cast # noqa: TID251 # native extension exposes untyped callables +from typing import TYPE_CHECKING, Final, Literal, Protocol, cast # noqa: TID251 # PyO3 binding validation from pydantic import TypeAdapter +from typing_extensions import assert_never import litellm -from litellm._logging import verbose_logger -from litellm.litellm_core_utils.default_encoding import cl100k_base_rank_file, o200k_base_rank_file -from litellm.litellm_core_utils.token_counter import openai_tokenizer_encoding, uses_legacy_message_accounting +from litellm.litellm_core_utils.token_counter import openai_tokenizer_encoding_name, uses_legacy_message_accounting +from litellm.rust_bridge import tokenizer as tokenizer_dispatch from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.configuration import rust_enabled -from litellm.rust_bridge.runtime import BridgeErrorContext, RustHandled, aattempt -from litellm.utils import claude_json_str, huggingface_tokenizer_kind +from litellm.utils import huggingface_tokenizer_kind + +if TYPE_CHECKING: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer RustTokenizer = Literal["anthropic", "cl100k_base", "o200k_base"] @@ -27,13 +28,7 @@ class RustTokenCounter(Protocol): class RustTokenCounterFactory(Protocol): - def __call__(self, tokenizer_json: str) -> RustTokenCounter: - raise NotImplementedError - - def from_cl100k_ranks(self, rank_file: str) -> RustTokenCounter: - raise NotImplementedError - - def from_o200k_ranks(self, rank_file: str) -> RustTokenCounter: + def from_tokenizer(self, tokenizer: NativeTokenizer, fast: bool = False) -> RustTokenCounter: raise NotImplementedError @@ -51,7 +46,7 @@ def _as_factory(value: object) -> RustTokenCounterFactory | None: cast( # cast-ok: native extension protocol is runtime-defined RustTokenCounterFactory, value ) - if callable(value) + if callable(getattr(value, "from_tokenizer", None)) else None ) @@ -73,7 +68,7 @@ def rust_tokenizer(model: str) -> RustTokenizer | None: return "anthropic" if kind is not None or uses_legacy_message_accounting(model): return None - match openai_tokenizer_encoding(model).name: + match openai_tokenizer_encoding_name(model): case "cl100k_base": return "cl100k_base" case "o200k_base": @@ -84,31 +79,26 @@ def rust_tokenizer(model: str) -> RustTokenizer | None: @lru_cache(maxsize=4) def _counter(factory: RustTokenCounterFactory, tokenizer: RustTokenizer) -> RustTokenCounter: + return factory.from_tokenizer(_native_tokenizer(tokenizer)) + + +def _native_tokenizer(tokenizer: RustTokenizer) -> NativeTokenizer: match tokenizer: case "anthropic": - return factory(claude_json_str) - case "cl100k_base": - return factory.from_cl100k_ranks(cl100k_base_rank_file()) - case "o200k_base": - return factory.from_o200k_ranks(o200k_base_rank_file()) + native = tokenizer_dispatch.native_anthropic() + case "cl100k_base" | "o200k_base": + native = tokenizer_dispatch.native_encoding(tokenizer) + case _: + assert_never(tokenizer) + if native is None: + raise RuntimeError(f"native {tokenizer} tokenizer is unavailable") + return native -async def count_input_tokens(body: bytes, tokenizer: RustTokenizer) -> InputTokenCount | None: - if not rust_enabled(): - return None - factory: Final = TOKEN_COUNTER.load() - if factory is None: - return None - try: - attempt: Final = await aattempt( - native_call=lambda: _counter(factory, tokenizer).acount_request(body), - adapt=_INPUT_TOKEN_COUNT.validate_python, - context=BridgeErrorContext(route="token_counter", provider=tokenizer, model=""), - ) - except (RuntimeError, ValueError) as error: - verbose_logger.debug("Rust token counter (%s) failed, counting in Python: %s", tokenizer, error) - return None - if not isinstance(attempt, RustHandled): - return None - verbose_logger.debug("Rust token counter (%s) counted %d input tokens", tokenizer, attempt.value.input_tokens) - return attempt.value +async def native_count(factory: RustTokenCounterFactory, tokenizer: RustTokenizer, body: bytes) -> InputTokenCount: + """One native count, validated into the public shape. + + ``RustBridgeDeclined`` and upstream errors propagate so the caller's route + runner can map them onto its fallback policy; other failures (RuntimeError, + ValueError) propagate as-is.""" + return _INPUT_TOKEN_COUNT.validate_python(await _counter(factory, tokenizer).acount_request(body)) diff --git a/litellm/rust_bridge/tokenizer.py b/litellm/rust_bridge/tokenizer.py new file mode 100644 index 00000000000..5ed89813620 --- /dev/null +++ b/litellm/rust_bridge/tokenizer.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from functools import lru_cache +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # native class is validated at the binding boundary + +import tiktoken +from tokenizers import Tokenizer as PythonHuggingFaceTokenizer + +from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, HuggingFaceTokenizer, OpenAIEncoding +from litellm.rust_bridge import runtime +from litellm.rust_bridge.bindings import NativeBinding +from litellm.rust_bridge.catalog import Route, RouteContext + +if TYPE_CHECKING: + from litellm.rust_bridge._native import Tokenizer as NativeTokenizer + + +def _as_factory(value: object) -> type[NativeTokenizer] | None: + return ( + cast(type["NativeTokenizer"], value) # cast-ok: PyO3 class validated at the native boundary + if isinstance(value, type) + else None + ) + + +TOKENIZER: Final = NativeBinding("Tokenizer", validate=_as_factory) + +# The catalog contexts the tokenizer factories dispatch on. Callers that cache a tokenizer per +# backend key their cache on `decision(...)` of the same context, so key and dispatch agree. +TIKTOKEN_CONTEXT: Final = RouteContext(Route.TOKENIZER, provider="tiktoken") +HUGGINGFACE_CONTEXT: Final = RouteContext(Route.TOKENIZER, provider="huggingface") + + +@lru_cache(maxsize=8) +def _native_tiktoken(factory: type[NativeTokenizer], name: str) -> NativeTokenizer: + return factory.from_tiktoken(name) + + +@lru_cache(maxsize=1) +def _native_anthropic(factory: type[NativeTokenizer]) -> NativeTokenizer: + from litellm.utils import claude_json_str + + return factory.from_json(claude_json_str) + + +@lru_cache(maxsize=8) +def _native_encoding(factory: type[NativeTokenizer], name: str) -> OpenAIEncoding: + return OpenAIEncoding.wrap(_native_tiktoken(factory, name)) + + +def native_encoding(name: str) -> NativeTokenizer | None: + """The native tiktoken encoding behind `get_encoding(name)`, for a Rust route that counts + with the same loaded model; `None` without the extension.""" + factory: Final = TOKENIZER.load() + return None if factory is None else _native_tiktoken(factory, name) + + +def native_anthropic() -> NativeTokenizer | None: + """The native packaged Anthropic tokenizer behind `anthropic()`, parsed once per process.""" + factory: Final = TOKENIZER.load() + return None if factory is None else _native_anthropic(factory) + + +def _python_encoding(name: str) -> tiktoken.Encoding: + from litellm.litellm_core_utils.default_encoding import encoding + + return encoding if name == encoding.name else tiktoken.get_encoding(name) + + +def get_encoding(name: str) -> Encoding: + return runtime.run( + TIKTOKEN_CONTEXT, + binding=TOKENIZER, + native=lambda factory: _native_encoding(factory, name), + python=lambda: _python_encoding(name), + ) + + +def anthropic() -> HuggingFace: + """The packaged Anthropic tokenizer on the selected backend.""" + from litellm.utils import claude_json_str + + return runtime.run( + HUGGINGFACE_CONTEXT, + binding=TOKENIZER, + native=lambda factory: HuggingFaceTokenizer(_native_anthropic(factory)), + python=lambda: PythonHuggingFaceTokenizer.from_str(claude_json_str), + ) + + +def from_str(json: str) -> HuggingFace: + return runtime.run( + HUGGINGFACE_CONTEXT, + binding=TOKENIZER, + native=lambda factory: HuggingFaceTokenizer(factory.from_json(json)), + python=lambda: PythonHuggingFaceTokenizer.from_str(json), + ) + + +def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> HuggingFace: + return runtime.run( + HUGGINGFACE_CONTEXT, + binding=TOKENIZER, + native=lambda factory: HuggingFaceTokenizer( + factory.from_pretrained(identifier, revision=revision, token=token) + ), + python=lambda: PythonHuggingFaceTokenizer.from_pretrained(identifier, revision=revision, token=token), + ) diff --git a/litellm/secret_managers/aws_secret_manager_v2.py b/litellm/secret_managers/aws_secret_manager_v2.py index 0fb59105b5f..3e9f4e259d5 100644 --- a/litellm/secret_managers/aws_secret_manager_v2.py +++ b/litellm/secret_managers/aws_secret_manager_v2.py @@ -16,6 +16,8 @@ Requires: import json import os +from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Final import httpx @@ -294,32 +296,13 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): raise ValueError("Tags must be a dict or list of {Key, Value} pairs") data["Tags"] = tags_list - endpoint_url, headers, body = self._prepare_request( - action="CreateSecret", + create_response: Final = await self._async_create_or_restore_secret( secret_name=secret_name, - secret_value=secret_value, - optional_params=optional_params, request_data=data, + optional_params=optional_params, + timeout=timeout, ) - async_client: Final = get_async_httpx_client( - llm_provider=httpxSpecialProvider.SecretManager, - params={"timeout": timeout}, - ) - - try: - response: Final = await async_client.post( - url=endpoint_url, - headers=headers, - data=body.decode("utf-8"), - ) - response.raise_for_status() - create_response: Final = response.json() - except httpx.HTTPStatusError as err: - raise ValueError(f"HTTP error occurred: {err.response.text}") - except httpx.TimeoutException: - raise ValueError("Timeout error occurred") - if self.replica_regions: try: await self.async_replicate_secret( @@ -343,6 +326,110 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager): return create_response + async def _async_create_or_restore_secret( + self, + secret_name: str, + request_data: Mapping[str, object], + optional_params: dict | None, + timeout: float | httpx.Timeout | None, + ) -> dict[str, object]: + try: + return await self._async_post_action( + action="CreateSecret", + secret_name=secret_name, + request_data=request_data, + optional_params=optional_params, + timeout=timeout, + ) + except ValueError: + if not await self._async_is_scheduled_for_deletion( + secret_name=secret_name, + optional_params=optional_params, + timeout=timeout, + ): + raise + + verbose_logger.info( + "Secret %s is scheduled for deletion, restoring and updating in place (RestoreSecret + UpdateSecret)", + secret_name, + ) + await self._async_post_action( + action="RestoreSecret", + secret_name=secret_name, + request_data=None, + optional_params=optional_params, + timeout=timeout, + ) + update_data: Final = MappingProxyType( + {("SecretId" if key == "Name" else key): value for key, value in request_data.items() if key != "Tags"} + ) + tags: Final = request_data.get("Tags") + try: + updated: Final = await self._async_post_action( + action="UpdateSecret", + secret_name=secret_name, + request_data=update_data, + optional_params=optional_params, + timeout=timeout, + ) + if tags is not None: + await self._async_post_action( + action="TagResource", + secret_name=secret_name, + request_data=MappingProxyType({"SecretId": secret_name, "Tags": tags}), + optional_params=optional_params, + timeout=timeout, + ) + except ValueError: + await self.async_delete_secret(secret_name=secret_name, optional_params=optional_params, timeout=timeout) + raise + return updated + + async def _async_is_scheduled_for_deletion( + self, + secret_name: str, + optional_params: dict | None, + timeout: float | httpx.Timeout | None, + ) -> bool: + try: + described: Final = await self._async_post_action( + action="DescribeSecret", + secret_name=secret_name, + request_data=None, + optional_params=optional_params, + timeout=timeout, + ) + except ValueError: + return False + return described.get("DeletedDate") is not None + + async def _async_post_action( + self, + action: str, + secret_name: str, + request_data: Mapping[str, object] | None, + optional_params: dict | None, + timeout: float | httpx.Timeout | None, + ) -> dict[str, object]: + endpoint_url, headers, body = self._prepare_request( + action=action, + secret_name=secret_name, + optional_params=optional_params, + request_data=dict(request_data) if request_data is not None else None, + ) + async_client: Final = get_async_httpx_client( + llm_provider=httpxSpecialProvider.SecretManager, + params={"timeout": timeout}, + ) + try: + response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8")) + response.raise_for_status() + return response.json() + except httpx.HTTPStatusError as err: + raise ValueError(f"HTTP error occurred: {err.response.text}") + except httpx.TimeoutException: + raise ValueError("Timeout error occurred") + async def async_replicate_secret( self, secret_name: str, diff --git a/litellm/setup_wizard.py b/litellm/setup_wizard.py index dd147aaccee..8b946923cd2 100644 --- a/litellm/setup_wizard.py +++ b/litellm/setup_wizard.py @@ -53,13 +53,14 @@ PROVIDERS: Final[list[dict]] = [ { "id": "anthropic", "name": "Anthropic", - "description": "Claude Fable 5.1, Fable 5, Opus 5, Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 5, Sonnet 4.6, Haiku 4.5", + "description": "Claude Fable 5.1, Fable 5, Opus 5.5, Opus 5, Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 5, Sonnet 4.6, Haiku 4.5", "env_key": "ANTHROPIC_API_KEY", "key_hint": "sk-ant-...", "test_model": "claude-haiku-4-5-20251001", "models": [ "claude-fable-5-1", "claude-fable-5", + "claude-opus-5-5", "claude-opus-5", "claude-sonnet-5", "claude-opus-4-8", diff --git a/litellm/types/caching.py b/litellm/types/caching.py index 747f202d35e..6d42556a9c4 100644 --- a/litellm/types/caching.py +++ b/litellm/types/caching.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Any, Final, Literal, Optional, Union from pydantic import BaseModel -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict class LiteLLMCacheType(str, Enum): @@ -137,12 +137,16 @@ class HealthCheckCacheParams(BaseModel): redis_version: str | int | float | None = None +EMBEDDING_CACHE_FORMAT_VERSION: Final = 2 + + class CachedEmbedding(TypedDict): """Type definition for cached embedding objects""" - embedding: list[float] | None - index: int | None - object: str | None - model: str | None - prompt_tokens: int | None - prompt_tokens_details: dict | None + embedding: ReadOnly[list[float] | str | None] + index: ReadOnly[int | None] + object: ReadOnly[str | None] + model: ReadOnly[str | None] + prompt_tokens: ReadOnly[int | None] + prompt_tokens_details: ReadOnly[dict | None] + format_version: ReadOnly[int] diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index f4893e857d1..c929ee2ee79 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -131,6 +131,7 @@ EXCEPTION_STATUS: Final = "exception_status" EXCEPTION_CLASS: Final = "exception_class" RATE_LIMIT_CATEGORY: Final = "rate_limit_category" RATE_LIMIT_TYPE: Final = "rate_limit_type" +ZERO_COST_REASON_LABEL: Final = "reason" STATUS_CODE: Final = "status_code" EXCEPTION_LABELS: Final = [EXCEPTION_STATUS, EXCEPTION_CLASS] LATENCY_BUCKETS: Final = ( @@ -279,6 +280,7 @@ DEFINED_PROMETHEUS_METRICS = Literal[ "litellm_guardrail_latency_seconds", "litellm_guardrail_errors_total", "litellm_guardrail_requests_total", + "litellm_zero_cost_requests_total", # Cache metrics "litellm_cache_hits_metric", "litellm_cache_misses_metric", @@ -590,6 +592,14 @@ class PrometheusMetricLabels: UserAPIKeyLabelNames.SERVICE_TIER.value, ] + litellm_zero_cost_requests_total = ( + UserAPIKeyLabelNames.REQUESTED_MODEL.value, + UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value, + UserAPIKeyLabelNames.MODEL_ID.value, + UserAPIKeyLabelNames.API_PROVIDER.value, + ZERO_COST_REASON_LABEL, + ) + litellm_input_tokens_metric = [ UserAPIKeyLabelNames.END_USER.value, UserAPIKeyLabelNames.API_KEY_HASH.value, diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index b38684f1856..042df6f37fa 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -216,11 +216,20 @@ class AnthropicMessagesToolUseParam(TypedDict, total=False): caller: ToolCaller | None +class CompactionBlock(TypedDict, total=False): + """Native compaction block, signed for on-demand compaction.""" + + type: Required[ReadOnly[Literal["compaction"]]] + content: ReadOnly[str | None] + signature: ReadOnly[str] + + AnthropicMessagesAssistantMessageValues = ( AnthropicMessagesTextParam | AnthropicMessagesToolUseParam | ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock + | CompactionBlock ) @@ -390,6 +399,11 @@ AllAnthropicPassThroughMessageValues: TypeAlias = ( ) +class AnthropicCompaction(TypedDict, total=False): + type: Required[ReadOnly[Literal["summarize"]]] + instructions: ReadOnly[str] + + class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): max_tokens: int | None metadata: AnthropicMetadata | dict | None @@ -405,6 +419,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): top_p: float | None mcp_servers: list[AnthropicMcpServerTool] | None context_management: dict[str, Any] | None + compaction: ReadOnly[AnthropicCompaction | None] container: dict[str, Any] | None # Container config with skills for code execution output_format: AnthropicOutputSchema | None # Structured outputs support speed: str | None # Fast mode support for Opus models @@ -566,13 +581,6 @@ class ContextManagementResponse(TypedDict, total=False): applied_edits: list[AppliedEdit] -class CompactionBlock(TypedDict, total=False): - """Synthesized ``compaction`` content block (compact_20260112).""" - - type: Required[Literal["compaction"]] - content: str | None - - class UsageIteration(TypedDict, total=False): """One sampling iteration's token usage (compact_20260112).""" @@ -746,6 +754,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): WEB_SEARCH_2025_03_05 = "web-search-2025-03-05" CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27" COMPACT_2026_01_12 = "compact-2026-01-12" + COMPACT_2026_09_04 = "compact-2026-09-04" STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13" ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20" FAST_MODE_2026_02_01 = "fast-mode-2026-02-01" diff --git a/litellm/types/llms/anthropic_messages/anthropic_response.py b/litellm/types/llms/anthropic_messages/anthropic_response.py index 1d4c3cdc864..5cf988bd19e 100644 --- a/litellm/types/llms/anthropic_messages/anthropic_response.py +++ b/litellm/types/llms/anthropic_messages/anthropic_response.py @@ -1,3 +1,4 @@ +from collections.abc import Sequence from typing import Any, Literal, TypeAlias from typing_extensions import NotRequired, ReadOnly, TypedDict @@ -6,8 +7,10 @@ from litellm.types.llms.anthropic import ( AnthropicResponseContentBlockText, AnthropicResponseContentBlockToolUse, AnthropicStopDetails, + CompactionBlock, ContextManagementResponse, ServerToolUsage, + UsageIteration, ) @@ -56,6 +59,7 @@ AnthropicResponseContentBlock: TypeAlias = ( | AnthropicResponseToolUseBlock | AnthropicResponseThinkingBlock | AnthropicResponseRedactedThinkingBlock + | CompactionBlock ) @@ -66,6 +70,7 @@ class AnthropicUsage(TypedDict, total=False): input_tokens: int output_tokens: int + iterations: ReadOnly[Sequence[UsageIteration]] """ Cache Tokens Used @@ -91,7 +96,9 @@ class AnthropicMessagesResponse(TypedDict, total=False): id: str model: str | None # This represents the Model type from Anthropic role: Literal["assistant"] | None - stop_reason: Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal"] | None + stop_reason: ReadOnly[ + Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"] | None + ] stop_details: NotRequired[ReadOnly[AnthropicStopDetails | None]] stop_sequence: str | None type: Literal["message"] | None diff --git a/litellm/types/llms/custom_http.py b/litellm/types/llms/custom_http.py index 793893451df..06982a16755 100644 --- a/litellm/types/llms/custom_http.py +++ b/litellm/types/llms/custom_http.py @@ -32,6 +32,7 @@ class httpxSpecialProvider(str, Enum): Sandbox = "sandbox" ModelCostMap = "model_cost_map" PasswordBreachCheck = "password_breach_check" + ASGI = "asgi" VerifyTypes = str | bool | ssl.SSLContext diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index bc44eb5b5b7..599b1a76249 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -110,6 +110,14 @@ FileTypes = ( EmbeddingInput = str | list[str] +class BinaryResponseSummary(TypedDict): + """What logging keeps of a binary response (speech audio, file content): size and media type, never the bytes.""" + + object: ReadOnly[Literal["binary"]] + content_type: ReadOnly[str | None] + num_bytes: ReadOnly[int] + + class HttpxBinaryResponseContent(_HttpxBinaryResponseContent): _hidden_params: dict @@ -117,6 +125,19 @@ class HttpxBinaryResponseContent(_HttpxBinaryResponseContent): super().__init__(response) self._hidden_params = {} # mutable-ok: mutable-dict contract shared with ModelResponse logging consumers + def logging_summary(self) -> BinaryResponseSummary: + return { + "object": "binary", + "content_type": self.response.headers.get("content-type"), + "num_bytes": self._num_bytes(), + } + + def _num_bytes(self) -> int: + try: + return len(self.response.content) + except httpx.ResponseNotRead: + return self.response.num_bytes_downloaded + def set_response_cost(self, response_cost: float | None) -> None: if response_cost is None: self._hidden_params.pop("response_cost", None) diff --git a/litellm/types/management_endpoints/router_settings_endpoints.py b/litellm/types/management_endpoints/router_settings_endpoints.py index cef180b202a..4d1ac58edb0 100644 --- a/litellm/types/management_endpoints/router_settings_endpoints.py +++ b/litellm/types/management_endpoints/router_settings_endpoints.py @@ -80,6 +80,7 @@ class RouterSettingsField(BaseModel): # Routing strategy descriptions ROUTING_STRATEGY_DESCRIPTIONS: Final[dict[str, str]] = { + "priority": "Routes group calls to the lowest-priority-number available model, with failover to higher numbers. Equal priorities share traffic. Direct member calls keep their existing policy.", "simple-shuffle": "Randomly picks a deployment from the list. Simple and fast.", "least-busy": "Routes to the deployment with the lowest number of ongoing requests.", "latency-based-routing": "Routes to the deployment with the lowest latency over a sliding window.", diff --git a/litellm/types/passthrough_endpoints/pass_through_endpoints.py b/litellm/types/passthrough_endpoints/pass_through_endpoints.py index e47acf9d68b..619001a5791 100644 --- a/litellm/types/passthrough_endpoints/pass_through_endpoints.py +++ b/litellm/types/passthrough_endpoints/pass_through_endpoints.py @@ -28,6 +28,7 @@ class EndpointType(str, Enum): GEMINI = "gemini" ANTHROPIC = "anthropic" OPENAI = "openai" + TINYFISH = "tinyfish" GENERIC = "generic" diff --git a/litellm/types/passthrough_endpoints/tinyfish.py b/litellm/types/passthrough_endpoints/tinyfish.py new file mode 100644 index 00000000000..365eaef77a6 --- /dev/null +++ b/litellm/types/passthrough_endpoints/tinyfish.py @@ -0,0 +1,55 @@ +from typing import Final + +from typing_extensions import ReadOnly, TypedDict + +TINYFISH_AGENT_DEFAULT_API_BASE: Final = "https://agent.tinyfish.ai" +TINYFISH_AGENT_DOCS_URL: Final = "https://docs.tinyfish.ai/agent-api" +# TinyFish's published Agent API rate (USD per run step); override with env TINYFISH_COST_PER_STEP +TINYFISH_DEFAULT_COST_PER_STEP: Final = 0.016 +TINYFISH_MODEL_NAME: Final = "tinyfish/automation-run" +TINYFISH_POLLING_INTERVAL_SECONDS: Final = 5.0 +# the Agent API caps runs at 1200s but queue wait extends wall time, so billing polls with generous headroom +TINYFISH_MAX_POLLING_SECONDS: Final = 3600.0 +# at the 5s interval this tolerates a ~60s upstream outage before abandoning the charge +TINYFISH_MAX_CONSECUTIVE_POLL_FAILURES: Final = 12 + +TINYFISH_TERMINAL_RUN_STATUSES: Final = frozenset({"COMPLETED", "FAILED", "CANCELLED"}) + +# these fields use the shared account's saved logins/vault, so they 403 unless TINYFISH_ALLOW_AUTHENTICATED_RUNS=true +TINYFISH_AUTHENTICATED_RUN_FIELDS: Final = frozenset({"use_profile", "profile_id", "use_vault", "credential_item_ids"}) + +# litellm's pass-through envelope controls; rejected here or custom_body smuggles past the field gate and +# a caller stream flag flips the billing mode away from what the endpoint dictates +TINYFISH_REJECTED_ENVELOPE_FIELDS: Final = frozenset({"custom_body", "stream", "query_params"}) + +# covers the upstream 1200s max run duration plus response headroom for blocking runs +TINYFISH_PASSTHROUGH_TIMEOUT_SECONDS: Final = 1500.0 + +_RUN_SUBMIT_PATHS: Final = frozenset( + {("v1", "automation", "run"), ("v1", "automation", "run-async"), ("v1", "automation", "run-sse")} +) + + +class TinyfishRun(TypedDict, total=False): + """Run objects are null-heavy until terminal, so every field must tolerate None.""" + + run_id: ReadOnly[str | None] + status: ReadOnly[str | None] + num_of_steps: ReadOnly[int | None] + result: ReadOnly[object] + # left untyped on purpose: a strict error shape would fail whole-run validation on upstream drift and drop the charge + error: ReadOnly[object] + type: ReadOnly[str | None] + + +def is_allowed_tinyfish_endpoint(method: str, path: str) -> bool: + """The host also serves vault/wallet/profile management under the same key, so only run endpoints forward.""" + segments: Final = tuple(path.split("/")[1:]) + if not path.startswith("/") or any(segment in ("", ".", "..") for segment in segments): + return False + if method == "POST" and segments in _RUN_SUBMIT_PATHS: + return True + # no GET /v1/runs listing: run ids are unguessable, so blocking the list keeps teams out of each other's runs + if method == "GET" and len(segments) == 3 and segments[:2] == ("v1", "runs"): + return True + return method == "POST" and len(segments) == 4 and segments[:2] == ("v1", "runs") and segments[3] == "cancel" diff --git a/litellm/types/router.py b/litellm/types/router.py index 8ca27a9fb66..57bd4263894 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -6,7 +6,8 @@ import datetime import enum from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, ClassVar, Final, Generic, Literal, TypeVar, get_type_hints +from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Final, Generic, Literal, TypeVar, get_type_hints +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator @@ -15,6 +16,7 @@ from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_c from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.provider_affinity import validate_provider_affinity_header_name from litellm.types.router_weights import RouterWeights if TYPE_CHECKING: @@ -59,6 +61,25 @@ class RoutingGroup(BaseModel): routing_strategy: str routing_strategy_args: dict | None = None + model_priorities: dict[str, Annotated[int, Field(strict=True, ge=1, le=9007199254740991)]] | None = Field( + default=None, + description="For priority groups, every model's priority. Lower numbers are tried first; equal numbers share traffic.", + ) + + @model_validator(mode="after") + def _validate_model_priorities(self) -> "RoutingGroup": + if self.routing_strategy != "priority": + if self.model_priorities: + raise ValueError("model_priorities requires routing_strategy='priority'") + return self + if not self.models or len(self.models) != len(frozenset(self.models)): + raise ValueError("Priority routing groups require nonempty, distinct models") + if self.model_priorities is None or frozenset(self.model_priorities) != frozenset(self.models): + raise ValueError("model_priorities must contain exactly the group's models") + if self.routing_strategy_args: + raise ValueError("Priority routing groups use model_priorities, not routing_strategy_args") + return self + model_config = ConfigDict(protected_namespaces=()) @@ -163,6 +184,44 @@ def _as_utc(value: datetime.datetime | None) -> datetime.datetime | None: return value.astimezone(datetime.timezone.utc) +class ModelAccessWindow(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + start: datetime.time + end: datetime.time + timezone: str + team_ids: tuple[str, ...] = Field(min_length=1) + + @field_validator("start", "end") + @classmethod + def _naive_wall_clock(cls, value: datetime.time) -> datetime.time: + if value.tzinfo is not None: + raise ValueError("start and end must be local wall-clock times without a UTC offset") + return value + + @field_validator("timezone") + @classmethod + def _known_iana_timezone(cls, value: str) -> str: + try: + ZoneInfo(value) + except (ZoneInfoNotFoundError, ValueError) as exc: + raise ValueError(f"unknown IANA timezone '{value}'") from exc + return value + + @field_validator("team_ids") + @classmethod + def _non_empty_team_ids(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if any(not team_id for team_id in value): + raise ValueError("team_ids entries must be non-empty") + return value + + @model_validator(mode="after") + def _start_differs_from_end(self) -> "ModelAccessWindow": + if self.start == self.end: + raise ValueError("start and end must differ") + return self + + class ModelInfo(MirroredPricingParams): id: str | None # Allow id to be optional on input, but it will always be present as a str in the model instance db_model: bool = False # used for proxy - to separate models which are stored in the db vs. config. @@ -188,6 +247,8 @@ class ModelInfo(MirroredPricingParams): # admin-toggled pause flag; mirrors LiteLLM_ProxyModelTable.blocked blocked: bool | None = None + access_windows: tuple[ModelAccessWindow, ...] | None = None + # Bounds live on the model rather than litellm.constants: names there reach # litellm/__init__ through several modules' star re-exports, and a Final rebound that # way trips the basedpyright gate. @@ -336,6 +397,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): organization: str | None = None # for openai orgs configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None litellm_credential_name: str | None = None + provider_affinity_header: str | None = None ## LOGGING PARAMS ## litellm_trace_id: str | None = None @@ -407,6 +469,13 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): valkey_text_field: str | None = None valkey_embedding_field: str | None = None + @field_validator("provider_affinity_header") + @classmethod + def validate_provider_affinity_header(cls, value: str | None) -> str | None: + if value is None: + return None + return validate_provider_affinity_header_name(value) + @model_validator(mode="before") @classmethod def preprocess_input_data(cls, data: object) -> object: @@ -509,6 +578,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): stream_timeout: float | str | None max_retries: int | None organization: list | str | None # for openai orgs + provider_affinity_header: ReadOnly[str | None] configurable_clientside_auth_params: ( CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS # for allowing api base switching on finetuned models ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index e23f329ee83..f2c8f0e7044 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1,7 +1,7 @@ import json import re import time -from collections.abc import Mapping, Sequence +from collections.abc import Collection, Mapping, Sequence from enum import Enum from types import MappingProxyType from typing import ( @@ -36,12 +36,14 @@ from pydantic import ( BaseModel, ConfigDict, Field, + FieldSerializationInfo, JsonValue, PrivateAttr, SkipValidation, field_serializer, field_validator, ) +from pydantic.main import IncEx from typing_extensions import NotRequired, ReadOnly, Required, TypedDict from litellm._logging import verbose_logger @@ -80,7 +82,30 @@ from .llms.openai import ( ) from .rerank import RerankResponse as RerankResponse + +def _nested_selector( + selector: IncEx | None, + index: int, + count: int, + is_include: bool, +) -> tuple[bool, IncEx | None]: + if selector is None: + return True, None + if isinstance(selector, Mapping): + value: Final = selector.get(index, selector.get(index - count, selector.get("__all__"))) + keep: Final = value is not None if is_include else value is not True + per_item_selector: Final = None if value is True or value is None else value + return keep, per_item_selector + if isinstance(selector, Collection) and not isinstance(selector, (str, bytes)): + if all(isinstance(item, int) for item in selector): + addressed: Final = index in selector or index - count in selector + return (addressed if is_include else not addressed), None + return True, selector + + if TYPE_CHECKING: + from litellm.litellm_core_utils.tokenizer import Tokenizer + from .vector_stores import VectorStoreSearchResponse else: VectorStoreSearchResponse = Any @@ -173,6 +198,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_output_config: bool | None supports_image_size: bool | None supports_anthropic_thinking_payload: ReadOnly[bool | None] + supports_anthropic_compaction: ReadOnly[bool | None] supported_audio_formats: ReadOnly[Sequence[Literal["mp3", "wav"]] | None] vertex_ai_audio_api: ReadOnly[Literal["lyria_predict", "lyria_interactions"] | None] bedrock_output_config_effort_ceiling: Literal["low", "medium", "high", "max", "xhigh"] | None @@ -265,6 +291,10 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_read_input_token_cost_above_272k_tokens_priority: float | None cache_read_input_token_cost_above_272k_tokens_flex: float | None cache_read_input_token_cost_above_512k_tokens: float | None + cache_read_input_token_cost_batches: ReadOnly[float | None] + cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None] + cache_creation_input_token_cost_batches: ReadOnly[float | None] + cache_creation_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None] # Smallest prefix this model will actually cache, whatever caching mechanism its provider uses. # Absent means the provider-agnostic default applies; see MINIMUM_PROMPT_CACHE_TOKEN_COUNT. prompt_cache_min_tokens: int | None @@ -290,7 +320,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_second: float | None # for OpenAI Speech models input_cost_per_token_batches: float | None input_cost_per_video_token_batches: ReadOnly[float | None] + input_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None] output_cost_per_token_batches: float | None + output_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None] output_cost_per_token: Required[float | None] output_cost_per_token_flex: float | None # OpenAI flex service tier pricing output_cost_per_token_priority: float | None # OpenAI priority service tier pricing @@ -333,6 +365,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): output_cost_per_second_768p: ReadOnly[float | None] output_cost_per_second_2k: ReadOnly[float | None] output_cost_per_second_4k: ReadOnly[float | None] + output_cost_per_image_512: ReadOnly[float | None] + output_cost_per_image_1024: ReadOnly[float | None] + output_cost_per_image_1536: ReadOnly[float | None] ocr_cost_per_page: float | None # for OCR models ocr_cost_per_page_batches: ReadOnly[float | None] ocr_cost_per_credit: float | None # for OCR models priced by credit @@ -2557,8 +2592,35 @@ class ImageResponse(OpenAIImageResponse, BaseLiteLLMOpenAIResponseObject): model_config = ConfigDict(extra="allow", protected_namespaces=()) @field_serializer("data") - def _serialize_image_data(self, data: Sequence[OpenAIImage] | None) -> Sequence[Mapping[str, object]] | None: - return None if data is None else [image.model_dump() for image in data] + def _serialize_image_data( + self, + data: Sequence[OpenAIImage] | None, + info: FieldSerializationInfo, + ) -> Sequence[Mapping[str, object]] | None: + if data is None: + return None + include: Final = info.include + exclude: Final = info.exclude + + def _serialize_image(index: int, image: OpenAIImage) -> Mapping[str, object] | None: + include_keep, include_selector = _nested_selector(include, index, len(data), is_include=True) + exclude_keep, exclude_selector = _nested_selector(exclude, index, len(data), is_include=False) + if not include_keep or not exclude_keep: + return None + return image.model_dump( + mode=info.mode, + include=include_selector, + exclude=exclude_selector, + context=info.context, + exclude_none=info.exclude_none, + exclude_unset=info.exclude_unset, + exclude_defaults=info.exclude_defaults, + round_trip=info.round_trip, + by_alias=info.by_alias, + ) + + serialized_images: Final = tuple(_serialize_image(index, image) for index, image in enumerate(data)) + return [image for image in serialized_images if image is not None] def __init__( self, @@ -2970,6 +3032,7 @@ RoutingDecisionCause = Literal[ InternalCallOrigin = Literal[ "autorouter_classifier", + "autorouter_compaction", "shadow_eval_router", "shadow_eval_judge", "llm_as_a_judge_guardrail", @@ -3156,6 +3219,15 @@ class StandardLoggingModelCostFailureDebugInformation(TypedDict, total=False): custom_pricing: bool | None +ZeroCostReason = Literal["missing_pricing_key", "pricing_not_applied", "cost_calculation_error"] + + +class StandardLoggingZeroCostDiagnostic(TypedDict): + reason: ReadOnly[ZeroCostReason] + pricing_model: ReadOnly[str] + missing_pricing_keys: ReadOnly[tuple[str, ...]] + + class StandardLoggingPayloadErrorInformation(TypedDict, total=False): error_code: str | None error_class: str | None @@ -3474,6 +3546,7 @@ class StandardLoggingPayload(ClassifierAudit): autorouter_savings_estimate: ReadOnly[Mapping[str, JsonValue] | None] autorouter_baseline_observation: ReadOnly[str | None] response_cost_failure_debug_info: StandardLoggingModelCostFailureDebugInformation | None + zero_cost_diagnostic: NotRequired[ReadOnly[StandardLoggingZeroCostDiagnostic | None]] status: StandardLoggingPayloadStatus status_fields: StandardLoggingPayloadStatusFields custom_llm_provider: str | None @@ -3571,6 +3644,8 @@ class StandardCallbackDynamicParams(TypedDict, total=False): arize_api_key: str | None arize_space_key: str | None arize_space_id: str | None + arize_success_sampling_rate: ReadOnly[float | None] + arize_error_sampling_rate: ReadOnly[float | None] # PostHog dynamic params posthog_api_key: str | None @@ -3622,6 +3697,9 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): output_cost_per_second_768p: float | None = None output_cost_per_second_2k: float | None = None output_cost_per_second_4k: float | None = None + output_cost_per_image_512: float | None = None + output_cost_per_image_1024: float | None = None + output_cost_per_image_1536: float | None = None input_cost_per_pixel: float | None = None output_cost_per_pixel: float | None = None @@ -3646,6 +3724,10 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): cache_read_input_token_cost_above_200k_tokens_priority: float | None = None cache_read_input_token_cost_above_272k_tokens_priority: float | None = None cache_read_input_token_cost_above_272k_tokens_flex: float | None = None + cache_read_input_token_cost_batches: float | None = None + cache_read_input_token_cost_above_272k_tokens_batches: float | None = None + cache_creation_input_token_cost_batches: float | None = None + cache_creation_input_token_cost_above_272k_tokens_batches: float | None = None cache_read_input_audio_token_cost: float | None = None cache_read_input_image_token_cost: float | None = None input_cost_per_character_above_128k_tokens: float | None = None @@ -3656,6 +3738,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): input_cost_per_token_above_200k_tokens_priority: float | None = None input_cost_per_token_above_272k_tokens_priority: float | None = None input_cost_per_token_above_272k_tokens_flex: float | None = None + input_cost_per_token_above_272k_tokens_batches: float | None = None input_cost_per_query: float | None = None input_cost_per_image: float | None = None input_cost_per_image_above_128k_tokens: float | None = None @@ -3679,6 +3762,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): output_cost_per_token_above_200k_tokens_priority: float | None = None output_cost_per_token_above_272k_tokens_priority: float | None = None output_cost_per_token_above_272k_tokens_flex: float | None = None + output_cost_per_token_above_272k_tokens_batches: float | None = None output_cost_per_character_above_128k_tokens: float | None = None output_cost_per_image: float | None = None output_cost_per_image_token: float | None = None @@ -3792,6 +3876,24 @@ def echoed_cost_map_pricing_fields(model_info: Mapping[str, Any]) -> tuple[str, return tuple(sorted(k for k in model_info if is_server_derived_pricing_key(k))) +def echoed_cost_map_fields( + model_info: Mapping[str, object], *cost_map_entries: Mapping[str, object] +) -> tuple[str, ...]: + """Fields a ``/model/info`` echo copied from the cost map unchanged. + + Only ``litellm.get_model_info`` emits ``key``, so a blob carrying it is an echo of that + response. Anything in it that still equals a resolved cost-map entry is a display value + nobody typed; a value the operator edited differs from every entry and stays a real override. + Callers pass both the live entry, which the router rewrites with each deployment's own + overrides, and the catalog entry as loaded, so a reset to the catalog value reads as an echo either way. + """ + if COST_MAP_LOOKUP_KEY not in model_info: + return () + return tuple( + sorted(k for k, v in model_info.items() if any(k in entry and entry[k] == v for entry in cost_map_entries)) + ) + + def pricing_override_fields(*sources: Mapping[str, Any]) -> tuple[str, ...]: return tuple( sorted( @@ -3852,6 +3954,7 @@ all_litellm_params = ( agentic_loop_internal_litellm_params + [TRUSTED_CALLBACK_VARS_FIELD, ADDRESSED_RESPONSE_ID_FIELD, *bedrock_batch_litellm_params] + [ + "_context_compaction_state", "metadata", "litellm_metadata", "keepalive_seconds", @@ -3970,6 +4073,7 @@ all_litellm_params = ( "litellm_credential_name", "allowed_openai_params", "litellm_session_id", + "provider_affinity_header", "use_litellm_proxy", "use_chat_completions_api", "rust", @@ -4322,7 +4426,7 @@ class ProviderSpecificHeader(TypedDict): class SelectTokenizerResponse(TypedDict): type: Literal["openai_tokenizer", "huggingface_tokenizer"] - tokenizer: Any + tokenizer: ReadOnly["Tokenizer"] class LiteLLMFineTuningJob(FineTuningJob): diff --git a/litellm/utils.py b/litellm/utils.py index b8d210e4640..666e84221c0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -35,19 +35,17 @@ from importlib import resources from inspect import iscoroutine from io import StringIO from os.path import abspath, dirname, join +from pathlib import PurePath from types import MappingProxyType import dotenv import httpx import openai -import tiktoken from httpx import Proxy from httpx._utils import get_environment_proxies from openai.lib import _parsing, _pydantic from openai.types.chat.completion_create_params import ResponseFormat from pydantic import BaseModel -from tiktoken import Encoding -from tokenizers import Tokenizer import litellm import litellm.litellm_core_utils @@ -81,12 +79,20 @@ from litellm.constants import ( PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import ( + bind_budget_reservation_to_callbacks, + normalize_drop_params, + unbind_budget_reservation_from_callbacks, +) from litellm.litellm_core_utils.fallback_generalizations import ( match_capability_generalizations, match_fill_missing_generalizations, ) from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload +from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, strip_special_tokens +from litellm.rust_bridge import tokenizer as tokenizer_dispatch +from litellm.rust_bridge.catalog import decision +from litellm.rust_bridge.configuration import Decision _CachingHandlerResponse = None _LLMCachingHandler = None @@ -283,7 +289,9 @@ except (ImportError, AttributeError, TypeError): claude_json_str = json.dumps(json_data) import importlib.metadata from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast, get_args +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_checkable + +from typing_extensions import assert_never from litellm import utils as litellm_utils @@ -879,6 +887,32 @@ async def _run_success_deployment_hook_on_converted_chat_stream( ) +@runtime_checkable +class _NamedFile(Protocol): + @property + def name(self) -> object: ... + + +def _ocr_document_summary(document: object) -> str: + if not isinstance(document, Mapping): + return "default-message-value" + doc: Final = cast(Mapping[str, object], document) # cast-ok: ocr()/aocr() type the document as Mapping[str, object] + location: Final = doc.get("document_url", doc.get("image_url")) + if isinstance(location, str): + header, separator, payload = location.partition(",") + return f"{header} ({len(payload)} chars)" if separator and header.startswith("data:") else location + file_input: Final = doc.get("file") + mime_type: Final = doc.get("mime_type") + kind: Final = f"file ({mime_type})" if isinstance(mime_type, str) else "file" + if isinstance(file_input, PurePath): + return f"{kind} {file_input.name}" + if isinstance(file_input, bytes): + return f"{kind} {len(file_input)} bytes" + if isinstance(file_input, _NamedFile) and isinstance(file_input.name, str): + return f"{kind} {PurePath(file_input.name).name}" + return kind + + # Runs once per call to check if the user wants to send their data anywhere - PostHog/Sentry/Slack/etc. def function_setup( original_function: str, @@ -1119,6 +1153,17 @@ def function_setup( messages = args[0] if len(args) > 0 else kwargs["prompt"] elif call_type == CallTypes.rerank.value or call_type == CallTypes.arerank.value: messages = kwargs.get("query") + elif call_type in (CallTypes.search.value, CallTypes.asearch.value): + search_query: Final = args[0] if len(args) > 0 else kwargs.get("query") + messages = ( + "\n".join(part for part in search_query if isinstance(part, str)) + if isinstance(search_query, list) + else search_query + ) + elif call_type in (CallTypes.image_edit.value, CallTypes.aimage_edit.value): + messages = args[1] if len(args) > 1 else kwargs.get("prompt") + elif call_type in (CallTypes.ocr.value, CallTypes.aocr.value): + messages = _ocr_document_summary(args[1] if len(args) > 1 else kwargs.get("document")) elif call_type == CallTypes.atranscription.value or call_type == CallTypes.transcription.value: _file_obj: Final[FileTypes] = args[1] if len(args) > 1 else kwargs["file"] # Lazy import audio_utils.utils only when needed for transcription calls @@ -1880,6 +1925,8 @@ def client(original_function): # Type assertion: logging_obj is guaranteed to be non-None after function_setup assert logging_obj is not None, "logging_obj should not be None after function_setup" + if not _is_litellm_internal_call: + bind_budget_reservation_to_callbacks(logging_obj.litellm_params) kwargs["litellm_logging_obj"] = logging_obj modified_kwargs: Final = await async_pre_call_deployment_hook(kwargs, call_type) @@ -1931,20 +1978,6 @@ def client(original_function): elif _caching_handler_response.embedding_all_elements_cache_hit is True: return _caching_handler_response.final_embedding_cached_response - provider_kwargs: Final = ( - MappingProxyType( - { - **kwargs, - "input": list( # mutable-ok: LiteLLM embedding calls require a list input - _caching_handler_response.remaining_embedding_inputs - ), - } - ) - if _caching_handler_response is not None - and _caching_handler_response.remaining_embedding_inputs is not None - else kwargs - ) - if _llm_caching_handler.preset_cache_key is not None: logging_obj.litellm_params["preset_cache_key"] = _llm_caching_handler.preset_cache_key @@ -1982,13 +2015,19 @@ def client(original_function): print_verbose(f"Error while checking max token limit: {e}") # MODEL CALL + call_kwargs: Final = ( + {**kwargs, "input": _caching_handler_response.embedding_uncached_input} + if _caching_handler_response is not None + and _caching_handler_response.embedding_uncached_input is not None + else kwargs + ) try: - result = await original_function(*args, **provider_kwargs) + result = await original_function(*args, **call_kwargs) except Exception as deployment_error: _deployment_call_end_time = datetime.datetime.now() # noqa: DTZ005 # matches the naive datetimes this whole function already times start_time/end_time with try: await async_post_call_failure_deployment_hook( - request_data=kwargs, + request_data=call_kwargs, exception=deployment_error, call_type=call_type, ) @@ -2029,7 +2068,7 @@ def client(original_function): post_call_processing( original_response=result, model=model, - optional_params=kwargs, + optional_params=call_kwargs, original_function=original_function, rules_obj=rules_obj, ) @@ -2037,7 +2076,7 @@ def client(original_function): _call_type_enum: Final = _CALL_TYPE_ENUM_MAP.get(call_type) if _call_type_enum is not None: result = await async_post_call_success_deployment_hook( - request_data=kwargs, + request_data=call_kwargs, response=result, call_type=_call_type_enum, ) @@ -2046,7 +2085,7 @@ def client(original_function): await _llm_caching_handler.async_set_cache( result=result, original_function=original_function, - kwargs=provider_kwargs, + kwargs=call_kwargs, args=args, ) @@ -2100,6 +2139,7 @@ def client(original_function): # the failure hook ran, so a slow callback doesn't inflate the reported duration. end_time = _deployment_call_end_time if _deployment_call_end_time is not None else datetime.datetime.now() # noqa: DTZ005 # matches the naive datetimes this whole function already times start_time/end_time with if logging_obj and not _is_litellm_internal_call: + unbind_budget_reservation_from_callbacks(logging_obj.litellm_params) try: logging_obj.failure_handler( e, traceback_exception, start_time, end_time @@ -2266,17 +2306,27 @@ def _select_tokenizer(model: str, custom_tokenizer: CustomHuggingfaceTokenizer | identifier=custom_tokenizer["identifier"], revision=custom_tokenizer["revision"], auth_token=custom_tokenizer["auth_token"], + backend=_huggingface_tokenizer_backend(), ) return _select_tokenizer_helper(model=model) +def _huggingface_tokenizer_backend() -> Decision: + """The backend `tokenizer_dispatch.from_str` / `from_pretrained` will select right now. + + Cached HuggingFace tokenizers are keyed on it, so flipping `LITELLM_RUST` or + `litellm.rust(...)` reaches a fresh object instead of the other backend's.""" + return decision(tokenizer_dispatch.HUGGINGFACE_CONTEXT) + + @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) -def _select_custom_tokenizer_helper(identifier: str, revision: str, auth_token: str | None) -> SelectTokenizerResponse: +def _select_custom_tokenizer_helper( + identifier: str, revision: str, auth_token: str | None, backend: Decision +) -> SelectTokenizerResponse: verbose_logger.debug("Loading custom HuggingFace tokenizer %s (revision %s)", identifier, revision) return create_pretrained_tokenizer(identifier=identifier, revision=revision, auth_token=auth_token) -@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) def _select_tokenizer_helper(model: str) -> SelectTokenizerResponse: if litellm.disable_hf_tokenizer_download is True: return _return_openai_tokenizer(model) @@ -2286,6 +2336,10 @@ def _select_tokenizer_helper(model: str) -> SelectTokenizerResponse: if result is not None: return result except Exception as e: + from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted, ProcessReservedForForking + + if isinstance(e, (ForkedAfterNativeRuntimeStarted, ProcessReservedForForking)): + raise verbose_logger.debug("Error selecting tokenizer: %s", e) # default - tiktoken @@ -2320,19 +2374,26 @@ def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | None: kind: Final = huggingface_tokenizer_kind(model) if kind is None: return None - return {"type": "huggingface_tokenizer", "tokenizer": _load_huggingface_tokenizer(kind)} + return { + "type": "huggingface_tokenizer", + "tokenizer": _load_huggingface_tokenizer(kind, _huggingface_tokenizer_backend()), + } -def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind) -> Tokenizer: +@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) +def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decision) -> HuggingFace: + """One tokenizer per kind and backend; `backend` is the cache key, the dispatch re-derives it.""" match kind: case "cohere": - return Tokenizer.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer") + return tokenizer_dispatch.from_pretrained("Xenova/c4ai-command-r-v01-tokenizer") case "anthropic": - return Tokenizer.from_str(claude_json_str) + return tokenizer_dispatch.anthropic() case "llama2": - return Tokenizer.from_pretrained("hf-internal-testing/llama-tokenizer") + return tokenizer_dispatch.from_pretrained("hf-internal-testing/llama-tokenizer") case "llama3": - return Tokenizer.from_pretrained("Xenova/llama-3-tokenizer") + return tokenizer_dispatch.from_pretrained("Xenova/llama-3-tokenizer") + case _: + assert_never(kind) def encode(model="", text="", custom_tokenizer: dict | None = None): @@ -2348,15 +2409,13 @@ def encode(model="", text="", custom_tokenizer: dict | None = None): enc: The encoded text. """ tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model=model) - if isinstance(tokenizer_json["tokenizer"], Encoding): - enc = tokenizer_json["tokenizer"].encode(text, disallowed_special=()) - else: - enc = tokenizer_json["tokenizer"].encode(text) - # Normalize: HuggingFace Tokenizer.encode() returns an Encoding object; - # extract .ids so the return type is always List[int]. - if hasattr(enc, "ids"): - return enc.ids - return enc + if tokenizer_json["type"] == "openai_tokenizer": + openai_tokenizer: Final = cast( # cast-ok: [LIT006] caller's explicit type tag selects this interface + Encoding, tokenizer_json["tokenizer"] + ) + return openai_tokenizer.encode(text, disallowed_special=()) + encoded: Final = tokenizer_json["tokenizer"].encode(text) + return encoded.ids if hasattr(encoded, "ids") else encoded def decode( @@ -2375,26 +2434,12 @@ def decode( """ tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model=model) if tokenizer_json["type"] == "huggingface_tokenizer": - if skip_special_tokens: - tokens = _strip_huggingface_special_token_ids(tokenizer_json["tokenizer"], tokens) - dec = tokenizer_json["tokenizer"].decode(tokens, skip_special_tokens=skip_special_tokens) - return dec - dec = tokenizer_json["tokenizer"].decode(tokens) - return dec - - -def _strip_huggingface_special_token_ids(tokenizer: Tokenizer, tokens: Sequence[int]) -> Sequence[int]: - try: - added_tokens_decoder: Final = tokenizer.get_added_tokens_decoder() - except Exception: - return tokens - - special_token_ids: Final = { - token_id for token_id, added_token in added_tokens_decoder.items() if getattr(added_token, "special", False) - } - if not special_token_ids: - return tokens - return [token for token in tokens if token not in special_token_ids] + ids: Final = strip_special_tokens(tokenizer_json["tokenizer"], tokens) if skip_special_tokens else tokens + hf_tokenizer: Final = cast( # cast-ok: [LIT006] caller's explicit type tag selects this interface + HuggingFace, tokenizer_json["tokenizer"] + ) + return hf_tokenizer.decode(ids, skip_special_tokens=skip_special_tokens) + return tokenizer_json["tokenizer"].decode(tokens) def create_pretrained_tokenizer(identifier: str, revision="main", auth_token: str | None = None): @@ -2410,7 +2455,7 @@ def create_pretrained_tokenizer(identifier: str, revision="main", auth_token: st dict: A dictionary with the tokenizer and its type. """ - tokenizer: Final = Tokenizer.from_pretrained(identifier, revision=revision, token=auth_token) + tokenizer: Final = tokenizer_dispatch.from_pretrained(identifier, revision=revision, token=auth_token) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} @@ -2425,7 +2470,7 @@ def create_tokenizer(json: str): dict: A dictionary with the tokenizer and its type. """ - tokenizer: Final = Tokenizer.from_str(json) + tokenizer: Final = tokenizer_dispatch.from_str(json) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} @@ -4945,7 +4990,9 @@ def add_provider_specific_params_to_optional_params( **extra_body, } - dropped_keys: Final = EXTRA_BODY_ROUTING_KEYS | frozenset(additional_drop_params or ()) + dropped_keys: Final = EXTRA_BODY_ROUTING_KEYS | frozenset( + param for param in (additional_drop_params or ()) if isinstance(param, str) + ) processed_extra_body: Final = {k: v for k, v in initial_extra_body.items() if k not in dropped_keys} _ensure_extra_body_is_safe: Final = getattr(sys.modules[__name__], "_ensure_extra_body_is_safe") @@ -6046,6 +6093,14 @@ def _get_model_info_helper( cache_read_input_token_cost_flex=_model_info.get("cache_read_input_token_cost_flex", None), cache_read_input_token_cost_priority=_model_info.get("cache_read_input_token_cost_priority", None), cache_read_input_token_cost_ultrafast=_model_info.get("cache_read_input_token_cost_ultrafast", None), + cache_read_input_token_cost_batches=_model_info.get("cache_read_input_token_cost_batches"), + cache_read_input_token_cost_above_272k_tokens_batches=_model_info.get( + "cache_read_input_token_cost_above_272k_tokens_batches" + ), + cache_creation_input_token_cost_batches=_model_info.get("cache_creation_input_token_cost_batches"), + cache_creation_input_token_cost_above_272k_tokens_batches=_model_info.get( + "cache_creation_input_token_cost_above_272k_tokens_batches" + ), cache_creation_input_token_cost_above_1hr=_model_info.get( "cache_creation_input_token_cost_above_1hr", None ), @@ -6076,7 +6131,13 @@ def _get_model_info_helper( input_cost_per_video_per_second=_model_info.get("input_cost_per_video_per_second", None), input_cost_per_token_batches=_model_info.get("input_cost_per_token_batches"), input_cost_per_video_token_batches=_model_info.get("input_cost_per_video_token_batches", None), + input_cost_per_token_above_272k_tokens_batches=_model_info.get( + "input_cost_per_token_above_272k_tokens_batches" + ), output_cost_per_token_batches=_model_info.get("output_cost_per_token_batches"), + output_cost_per_token_above_272k_tokens_batches=_model_info.get( + "output_cost_per_token_above_272k_tokens_batches" + ), output_cost_per_token=_output_cost_per_token, output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None), output_cost_per_token_priority=_model_info.get("output_cost_per_token_priority", None), @@ -6162,6 +6223,7 @@ def _get_model_info_helper( supports_tool_search=_model_info.get("supports_tool_search", None), supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None), supports_anthropic_thinking_payload=_model_info.get("supports_anthropic_thinking_payload", None), + supports_anthropic_compaction=_model_info.get("supports_anthropic_compaction", None), supports_none_reasoning_effort=_model_info.get("supports_none_reasoning_effort", None), supports_minimal_reasoning_effort=_model_info.get("supports_minimal_reasoning_effort", None), supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None), @@ -8359,6 +8421,7 @@ class ProviderConfigManager: False, ), LlmProviders.EDENAI: (litellm.EdenAIChatConfig, False), + LlmProviders.FAL_AI: (litellm.FalAIChatConfig, False), LlmProviders.COMETAPI: (lambda: litellm.CometAPIConfig(), False), LlmProviders.DATAROBOT: (lambda: litellm.DataRobotConfig(), False), LlmProviders.GEMINI: (lambda: litellm.GoogleAIStudioGeminiConfig(), False), @@ -9588,9 +9651,9 @@ class ProviderConfigManager: return BlackForestLabsImageEditConfig() elif LlmProviders.FAL_AI == provider: - from litellm.llms.fal_ai.image_edit import FalAIImageEditConfig + from litellm.llms.fal_ai.image_edit import get_fal_ai_image_edit_config - return FalAIImageEditConfig() + return get_fal_ai_image_edit_config(model) elif LlmProviders.AZURE_AI == provider: from litellm.llms.azure_ai.image_edit import get_azure_ai_image_edit_config diff --git a/migrations/Dockerfile b/migrations/Dockerfile index c6d1b0cc46e..f34940c0ce0 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -67,6 +67,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --extra extra_proxy \ --python python3.13 +RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/ + COPY migrations/run.py /app/run.py # Pre-warm the Prisma binary cache so the Job pod doesn't reach the diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 80bb2bf9bd2..224129fbd25 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -53,13 +53,6 @@ "mode": "image_generation", "output_cost_per_image": 0.04 }, - "1024-x-1024/dall-e-2": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 1.9e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "1024-x-1024/max-steps/stability.stable-diffusion-xl-v1": { "litellm_provider": "bedrock", "max_input_tokens": 77, @@ -67,13 +60,6 @@ "mode": "image_generation", "output_cost_per_image": 0.08 }, - "256-x-256/dall-e-2": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 2.4414e-07, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "512-x-512/50-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, @@ -81,13 +67,6 @@ "mode": "image_generation", "output_cost_per_image": 0.018 }, - "512-x-512/dall-e-2": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 6.86e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "512-x-512/max-steps/stability.stable-diffusion-xl-v0": { "litellm_provider": "bedrock", "max_input_tokens": 77, @@ -578,17 +557,6 @@ "supports_vision": true, "supports_tool_choice": true }, - "amazon.nova-sonic-v1:0": { - "deprecation_date": "2026-09-14", - "input_cost_per_audio_token": 3.4e-06, - "input_cost_per_token": 6e-08, - "litellm_provider": "bedrock", - "mode": "realtime", - "output_cost_per_audio_token": 1.36e-05, - "output_cost_per_token": 2.4e-07, - "supports_audio_input": true, - "supports_audio_output": true - }, "amazon.nova-2-sonic-v1:0": { "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 3.3e-07, @@ -972,23 +940,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 - }, "anthropic.claude-3-opus-20240229-v1:0": { "input_cost_per_token": 1.5e-05, "litellm_provider": "bedrock", @@ -1004,23 +955,6 @@ "cache_read_input_token_cost": 1.5e-06, "cache_creation_input_token_cost": 1.875e-05 }, - "anthropic.claude-3-sonnet-20240229-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, "anthropic.claude-instant-v1": { "input_cost_per_token": 8e-07, "litellm_provider": "bedrock", @@ -1839,6 +1773,46 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "thinking_always_on": true, + "supports_forced_tool_use": false + }, "global.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1877,6 +1851,46 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "global.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "thinking_always_on": true, + "supports_forced_tool_use": false + }, "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1915,6 +1929,46 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "us.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_1hr": 8.8e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 4.4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "thinking_always_on": true, + "supports_forced_tool_use": false + }, "eu.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1952,6 +2006,46 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512 }, + "eu.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_1hr": 8.8e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 4.4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "au.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -1989,6 +2083,46 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512 }, + "au.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_1hr": 8.8e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 4.4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "jp.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -2026,6 +2160,46 @@ "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512 }, + "jp.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5.5e-06, + "cache_creation_input_token_cost_above_1hr": 8.8e-06, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 4.4e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_native_structured_output": false, + "supports_max_reasoning_effort": true, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 512, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -2982,60 +3156,6 @@ "supports_vision": true, "supports_tool_choice": true }, - "apac.anthropic.claude-3-5-sonnet-20240620-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, - "apac.anthropic.claude-3-5-sonnet-20241022-v2:0": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "apac.anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 - }, "apac.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, "cache_read_input_token_cost": 1.1e-07, @@ -3063,23 +3183,6 @@ "input_cost_per_token_batches": 5.5e-07, "output_cost_per_token_batches": 2.75e-06 }, - "apac.anthropic.claude-3-sonnet-20240229-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, "apac.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, @@ -3424,6 +3527,41 @@ "supports_max_reasoning_effort": true, "prompt_cache_min_tokens": 512 }, + "azure_ai/claude-opus-5-5": { + "supports_mid_conversation_system": true, + "input_cost_per_token": 4e-06, + "output_cost_per_token": 2e-05, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 + }, "azure_ai/claude-opus-4-8": { "deprecation_date": "2027-09-01", "supports_mid_conversation_system": true, @@ -3457,29 +3595,6 @@ "supports_max_reasoning_effort": true, "prompt_cache_min_tokens": 1024 }, - "azure_ai/claude-opus-4-1": { - "deprecation_date": "2026-08-05", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "azure_ai", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, "azure_ai/claude-sonnet-4-5": { "deprecation_date": "2026-10-19", "cache_creation_input_token_cost": 3.75e-06, @@ -4428,7 +4543,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 6.875e-08 }, "azure/eu/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-08, @@ -4467,11 +4583,13 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.375e-08 }, "azure/eu/gpt-5.1": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_batches": 6.875e-08, "cache_read_input_token_cost_priority": 2.75e-07, "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", @@ -4511,43 +4629,6 @@ "output_cost_per_token_priority": 2.2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/eu/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.375e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.375e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/eu/gpt-5.1-codex": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.375e-07, @@ -4646,7 +4727,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 2.75e-09 }, "azure/eu/o1-2024-12-17": { "cache_read_input_token_cost": 8.25e-06, @@ -4684,21 +4766,6 @@ "supports_prompt_caching": true, "supports_vision": false }, - "azure/eu/o1-preview-2024-09-12": { - "cache_read_input_token_cost": 8.25e-06, - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_vision": false - }, "azure/eu/o3-mini-2025-01-31": { "cache_read_input_token_cost": 6.05e-07, "deprecation_date": "2026-11-19", @@ -4814,6 +4881,7 @@ "azure/global/gpt-5.1": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -4853,43 +4921,6 @@ "output_cost_per_token_priority": 2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/global/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/global/gpt-5.1-codex": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, @@ -4965,19 +4996,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "azure/gpt-3.5-turbo-0125": { - "deprecation_date": "2025-03-31", - "input_cost_per_token": 5e-07, - "litellm_provider": "azure", - "max_input_tokens": 16384, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true - }, "azure/gpt-3.5-turbo-instruct-0914": { "input_cost_per_token": 1.5e-06, "litellm_provider": "azure_text", @@ -4997,32 +5015,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "azure/gpt-35-turbo-0125": { - "deprecation_date": "2025-05-31", - "input_cost_per_token": 5e-07, - "litellm_provider": "azure", - "max_input_tokens": 16384, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true - }, - "azure/gpt-35-turbo-1106": { - "deprecation_date": "2025-03-31", - "input_cost_per_token": 1e-06, - "litellm_provider": "azure", - "max_input_tokens": 16384, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 2e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true - }, "azure/gpt-35-turbo-16k": { "input_cost_per_token": 3e-06, "litellm_provider": "azure", @@ -5790,40 +5782,6 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "azure/gpt-realtime-2": { - "cache_read_input_audio_token_cost": 4e-07, - "cache_read_input_token_cost": 4e-07, - "deprecation_date": "2026-08-31", - "input_cost_per_audio_token": 3.2e-05, - "input_cost_per_image_token": 5e-06, - "input_cost_per_token": 4e-06, - "litellm_provider": "azure", - "max_input_tokens": 32000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 6.4e-05, - "output_cost_per_token": 2.4e-05, - "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", - "supported_endpoints": [ - "/v1/realtime" - ], - "supported_modalities": [ - "text", - "image", - "audio" - ], - "supported_output_modalities": [ - "text", - "audio" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "azure/gpt-realtime-2.1": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, @@ -6118,46 +6076,8 @@ "input_cost_per_token_batches": 6.25e-07, "output_cost_per_token_batches": 5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_minimal_reasoning_effort": true - }, - "azure/gpt-5.1-chat-2025-11-13": { - "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_priority": 2.5e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "input_cost_per_token_priority": 2.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "output_cost_per_token_priority": 2e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_native_streaming": true, - "supports_parallel_function_calling": false, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": false, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_batches": 6.25e-08 }, "azure/gpt-5.1-codex-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, @@ -6232,6 +6152,7 @@ "azure/gpt-5": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, "input_cost_per_token_batches": 6.25e-07, @@ -6305,74 +6226,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true - }, - "azure/gpt-5-chat": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "azure/gpt-5-chat-latest": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 6.25e-08 }, "azure/gpt-5-codex": { "cache_read_input_token_cost": 1.25e-07, @@ -6409,6 +6264,7 @@ "azure/gpt-5-mini": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, @@ -6482,11 +6338,13 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.25e-08 }, "azure/gpt-5-nano": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_batches": 2.5e-09, "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, "litellm_provider": "azure", @@ -6554,7 +6412,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 2.5e-09 }, "azure/gpt-5-pro": { "deprecation_date": "2027-04-07", @@ -6591,6 +6450,7 @@ "azure/gpt-5.1": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", @@ -6630,43 +6490,6 @@ "output_cost_per_token_priority": 2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/gpt-5.1-codex": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.25e-07, @@ -6766,6 +6589,7 @@ "azure/gpt-5.2": { "deprecation_date": "2027-06-08", "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_batches": 8.75e-08, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, "input_cost_per_token_batches": 8.75e-07, @@ -6841,79 +6665,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true - }, - "azure/gpt-5.2-chat": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "azure/gpt-5.2-chat-2025-12-11": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-05-13", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 8.75e-08 }, "azure/gpt-5.2-codex": { "cache_read_input_token_cost": 1.75e-07, @@ -6947,42 +6700,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "azure/gpt-5.3-chat": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "azure/gpt-5.3-codex": { "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, @@ -7100,9 +6817,11 @@ "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2.5e-07, "cache_read_input_token_cost_priority": 5e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06, + "cache_read_input_token_cost_batches": 1.3e-07, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_priority": 5e-06, @@ -7140,9 +6859,11 @@ "supports_vision": true, "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_above_272k_tokens_flex": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, "input_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_above_272k_tokens_flex": 1.125e-05, "output_cost_per_token_batches": 7.5e-06, "output_cost_per_token_flex": 7.5e-06, @@ -7154,6 +6875,8 @@ "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.75e-07, "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, + "cache_read_input_token_cost_batches": 1.43e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, "input_cost_per_token_priority": 5.5e-06, @@ -7189,8 +6912,10 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "input_cost_per_token_above_272k_tokens": 5.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.75e-06, "input_cost_per_token_batches": 1.375e-06, "output_cost_per_token_above_272k_tokens": 2.475e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.2375e-05, "output_cost_per_token_batches": 8.25e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, @@ -7200,6 +6925,8 @@ "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 2.75e-07, "cache_read_input_token_cost_above_272k_tokens": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, + "cache_read_input_token_cost_batches": 1.43e-07, "cache_read_input_token_cost_priority": 5.5e-07, "input_cost_per_token": 2.75e-06, "input_cost_per_token_priority": 5.5e-06, @@ -7235,8 +6962,10 @@ "supports_none_reasoning_effort": true, "default_reasoning_effort": "none", "input_cost_per_token_above_272k_tokens": 5.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.75e-06, "input_cost_per_token_batches": 1.375e-06, "output_cost_per_token_above_272k_tokens": 2.475e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.2375e-05, "output_cost_per_token_batches": 8.25e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, @@ -7294,7 +7023,11 @@ "output_cost_per_token_flex": 7.5e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, + "cache_read_input_token_cost_batches": 1.3e-07, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05 }, "azure/us/gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.75e-07, @@ -7340,7 +7073,11 @@ "output_cost_per_token_batches": 8.25e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, + "cache_read_input_token_cost_batches": 1.43e-07, + "input_cost_per_token_above_272k_tokens_batches": 2.75e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.2375e-05 }, "azure/eu/gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.75e-07, @@ -7386,7 +7123,11 @@ "output_cost_per_token_batches": 8.25e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.75e-07, + "cache_read_input_token_cost_batches": 1.43e-07, + "input_cost_per_token_above_272k_tokens_batches": 2.75e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.2375e-05 }, "azure/gpt-5.4-pro": { "deprecation_date": "2027-09-07", @@ -7394,6 +7135,7 @@ "cache_read_input_token_cost_above_272k_tokens": 6e-06, "input_cost_per_token": 3e-05, "input_cost_per_token_above_272k_tokens": 6e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "input_cost_per_token_above_272k_tokens_flex": 3e-05, "input_cost_per_token_batches": 1.5e-05, "input_cost_per_token_flex": 1.5e-05, @@ -7404,6 +7146,7 @@ "mode": "responses", "output_cost_per_token": 0.00018, "output_cost_per_token_above_272k_tokens": 0.00027, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "output_cost_per_token_above_272k_tokens_flex": 0.000135, "output_cost_per_token_batches": 9e-05, "output_cost_per_token_flex": 9e-05, @@ -7482,7 +7225,9 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135 }, "azure/gpt-5.6": { "cache_creation_input_token_cost": 6.25e-06, @@ -8625,11 +8370,14 @@ "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 2e-06, + "cache_read_input_token_cost_batches": 2.5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_above_272k_tokens_flex": 5e-06, "input_cost_per_token_priority": 1.25e-05, "input_cost_per_token_above_272k_tokens_priority": 2e-05, @@ -8642,6 +8390,7 @@ "mode": "chat", "output_cost_per_token": 3e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "output_cost_per_token_above_272k_tokens_priority": 9e-05, "output_cost_per_token_batches": 1.5e-05, @@ -8682,9 +8431,12 @@ "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, + "cache_read_input_token_cost_batches": 2.75e-07, "cache_read_input_token_cost_priority": 1.375e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, + "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, "input_cost_per_token_batches": 2.75e-06, "input_cost_per_token_priority": 1.375e-05, "output_cost_per_token": 3.3e-05, @@ -8695,6 +8447,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, "output_cost_per_token_batches": 1.65e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -8733,9 +8486,12 @@ "deprecation_date": "2027-10-26", "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, + "cache_read_input_token_cost_batches": 2.75e-07, "cache_read_input_token_cost_priority": 1.375e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, + "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, "input_cost_per_token_batches": 2.75e-06, "input_cost_per_token_priority": 1.375e-05, "output_cost_per_token": 3.3e-05, @@ -8746,6 +8502,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, "output_cost_per_token_batches": 1.65e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, @@ -8783,8 +8540,10 @@ "azure/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 2e-06, + "cache_read_input_token_cost_batches": 2.5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, @@ -8828,9 +8587,11 @@ "supports_vision": true, "supports_web_search": true, "deprecation_date": "2027-10-26", + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_above_272k_tokens_flex": 5e-06, "input_cost_per_token_batches": 2.5e-06, "input_cost_per_token_flex": 2.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_batches": 1.5e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, @@ -8889,11 +8650,17 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, + "cache_read_input_token_cost_batches": 2.5e-07, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05 }, "azure/us/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, + "cache_read_input_token_cost_batches": 2.75e-07, "cache_read_input_token_cost_priority": 1.375e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, @@ -8935,7 +8702,9 @@ "supports_vision": true, "supports_web_search": true, "deprecation_date": "2027-10-26", + "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, "output_cost_per_token_batches": 1.65e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, @@ -8988,11 +8757,17 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, + "cache_read_input_token_cost_batches": 2.75e-07, + "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.475e-05 }, "azure/eu/gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5.5e-07, "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, + "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, + "cache_read_input_token_cost_batches": 2.75e-07, "cache_read_input_token_cost_priority": 1.375e-06, "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, @@ -9034,7 +8809,9 @@ "supports_vision": true, "supports_web_search": true, "deprecation_date": "2027-10-26", + "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.475e-05, "output_cost_per_token_batches": 1.65e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, @@ -9087,7 +8864,11 @@ "supports_none_reasoning_effort": true, "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "cache_read_input_token_cost_above_272k_tokens_batches": 5.5e-07, + "cache_read_input_token_cost_batches": 2.75e-07, + "input_cost_per_token_above_272k_tokens_batches": 5.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 2.475e-05 }, "azure/gpt-5.5-pro": { "cache_read_input_token_cost": 3e-06, @@ -9176,6 +8957,7 @@ "azure/gpt-5.4-mini": { "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -9273,11 +9055,13 @@ "output_cost_per_token_flex": 2.25e-06, "output_cost_per_token_priority": 9e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "cache_read_input_token_cost_batches": 3.75e-08 }, "azure/gpt-5.4-nano": { "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2e-08, + "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "input_cost_per_token": 2e-07, "litellm_provider": "azure", @@ -9369,7 +9153,8 @@ "output_cost_per_token_batches": 6.25e-07, "output_cost_per_token_flex": 6.25e-07, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "cache_read_input_token_cost_batches": 1e-08 }, "azure/gpt-image-1": { "cache_read_input_token_cost": 1.25e-06, @@ -9772,39 +9557,6 @@ "supports_reasoning": true, "supports_vision": false }, - "azure/o1-preview": { - "cache_read_input_token_cost": 7.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_vision": false - }, - "azure/o1-preview-2024-09-12": { - "cache_read_input_token_cost": 7.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_vision": false - }, "azure/o3": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5e-07, @@ -10416,7 +10168,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 6.875e-08 }, "azure/us/gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.75e-08, @@ -10455,7 +10208,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.375e-08 }, "azure/us/gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5.5e-09, @@ -10491,11 +10245,13 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 2.75e-09 }, "azure/us/gpt-5.1": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_batches": 6.875e-08, "cache_read_input_token_cost_priority": 2.75e-07, "input_cost_per_token": 1.375e-06, "litellm_provider": "azure", @@ -10535,43 +10291,6 @@ "output_cost_per_token_priority": 2.2e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/us/gpt-5.1-chat": { - "cache_read_input_token_cost": 1.375e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 1.375e-06, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/us/gpt-5.1-codex": { "deprecation_date": "2027-05-15", "cache_read_input_token_cost": 1.375e-07, @@ -10672,21 +10391,6 @@ "supports_prompt_caching": true, "supports_vision": false }, - "azure/us/o1-preview-2024-09-12": { - "cache_read_input_token_cost": 8.25e-06, - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "max_input_tokens": 128000, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_vision": false - }, "azure/us/o3-2025-04-16": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, @@ -11188,18 +10892,6 @@ "/v1/images/edits" ] }, - "azure_ai/MAI-Image-2e": { - "deprecation_date": "2026-08-15", - "input_cost_per_token": 5e-06, - "litellm_provider": "azure_ai", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "output_cost_per_image_token": 1.95e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/images/generations" - ] - }, "azure_ai/MAI-Thinking-1": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 2e-06, @@ -11224,34 +10916,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "azure_ai/Llama-3.2-11B-Vision-Instruct": { - "deprecation_date": "2026-06-13", - "input_cost_per_token": 3.7e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 3.7e-07, - "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-11b-vision-instruct-offer?tab=Overview", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "azure_ai/Llama-3.2-90B-Vision-Instruct": { - "deprecation_date": "2026-06-13", - "input_cost_per_token": 2.04e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 2.04e-06, - "source": "https://marketplace.microsoft.com/en/marketplace/apps/metagenai.meta-llama-3-2-90b-vision-instruct-offer?tab=Overview", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "azure_ai/Llama-3.3-70B-Instruct": { "input_cost_per_token": 7.1e-07, "litellm_provider": "azure_ai", @@ -11300,18 +10964,6 @@ "output_cost_per_token": 3.7e-07, "supports_tool_choice": true }, - "azure_ai/Meta-Llama-3.1-405B-Instruct": { - "deprecation_date": "2026-06-13", - "input_cost_per_token": 5.33e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 1.6e-05, - "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-405b-instruct-offer?tab=PlansAndPrice", - "supports_tool_choice": true - }, "azure_ai/Meta-Llama-3.1-70B-Instruct": { "input_cost_per_token": 2.68e-06, "litellm_provider": "azure_ai", @@ -11323,18 +10975,6 @@ "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-70b-instruct-offer?tab=PlansAndPrice", "supports_tool_choice": true }, - "azure_ai/Meta-Llama-3.1-8B-Instruct": { - "deprecation_date": "2026-06-13", - "input_cost_per_token": 3e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 2048, - "max_tokens": 2048, - "mode": "chat", - "output_cost_per_token": 6.1e-07, - "source": "https://marketplace.microsoft.com/en-us/marketplace/apps/metagenai.meta-llama-3-1-8b-instruct-offer?tab=PlansAndPrice", - "supports_tool_choice": true - }, "azure_ai/Phi-3-medium-128k-instruct": { "input_cost_per_token": 1.7e-07, "litellm_provider": "azure_ai", @@ -11505,16 +11145,6 @@ "supports_tool_choice": true, "supports_reasoning": true }, - "azure_ai/mistral-document-ai-2505": { - "deprecation_date": "2026-07-20", - "litellm_provider": "azure_ai", - "ocr_cost_per_page": 0.003, - "mode": "ocr", - "supported_endpoints": [ - "/v1/ocr" - ], - "source": "https://devblogs.microsoft.com/foundry/whats-new-in-azure-ai-foundry-august-2025/#mistral-document-ai-(ocr)-%E2%80%94-serverless-in-foundry" - }, "azure_ai/mistral-document-ai-2512": { "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.003, @@ -11615,17 +11245,6 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, - "azure_ai/cohere-rerank-v3.5": { - "deprecation_date": "2026-05-14", - "input_cost_per_query": 0.002, - "input_cost_per_token": 0.0, - "litellm_provider": "azure_ai", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "rerank", - "output_cost_per_token": 0.0 - }, "azure_ai/cohere-rerank-v4.0-pro": { "input_cost_per_query": 0.0025, "input_cost_per_token": 0.0, @@ -11678,19 +11297,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "azure_ai/deepseek-r1": { - "deprecation_date": "2026-08-13", - "input_cost_per_token": 1.35e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 5.4e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_reasoning": true, - "supports_tool_choice": true - }, "azure_ai/deepseek-v3": { "input_cost_per_token": 1.14e-06, "litellm_provider": "azure_ai", @@ -11702,33 +11308,6 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_tool_choice": true }, - "azure_ai/deepseek-v3-0324": { - "deprecation_date": "2026-07-13", - "input_cost_per_token": 1.14e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 128000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4.56e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_tool_choice": true - }, - "azure_ai/deepseek-v3.1": { - "deprecation_date": "2026-07-13", - "input_cost_per_token": 1.23e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 4.94e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "azure_ai/deepseek-v4-pro": { "deprecation_date": "2028-02-20", "input_cost_per_token": 1.74e-06, @@ -11795,68 +11374,6 @@ ], "supports_embedding_image_input": true }, - "azure_ai/global/grok-3": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 3e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true - }, - "azure_ai/global/grok-3-mini": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.27e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true - }, - "azure_ai/grok-3": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 3e-06, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true - }, - "azure_ai/grok-3-mini": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.27e-06, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true - }, "azure_ai/grok-4": { "input_cost_per_token": 3e-06, "litellm_provider": "azure_ai", @@ -11950,36 +11467,6 @@ "supports_vision": true, "supports_web_search": true }, - "azure_ai/grok-4-fast-non-reasoning": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 2e-07, - "output_cost_per_token": 5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "azure_ai/grok-4-fast-reasoning": { - "deprecation_date": "2026-05-01", - "input_cost_per_token": 2e-07, - "output_cost_per_token": 5e-07, - "litellm_provider": "azure_ai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_web_search": true - }, "azure_ai/grok-4-1-fast-non-reasoning": { "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, @@ -12934,6 +12421,23 @@ "supports_tool_choice": true, "output_cost_per_token": 1.86e-06 }, + "bedrock/eu-west-2/nvidia.nemotron-super-3-120b": { + "input_cost_per_token": 2.3e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 256000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.01e-06, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, "litellm_provider": "bedrock", @@ -13486,40 +12990,6 @@ "mode": "chat", "output_cost_per_token": 1.5e-06 }, - "bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3.6e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3.6e-07, - "cache_creation_input_token_cost": 4.5e-06 - }, - "bedrock/us-gov-east-1/anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 3e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-08, - "cache_creation_input_token_cost": 3.75e-07 - }, "bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, @@ -13697,61 +13167,6 @@ "mode": "chat", "output_cost_per_token": 1.5e-06 }, - "bedrock/us-gov-west-1/anthropic.claude-3-7-sonnet-20250219-v1:0": { - "cache_creation_input_token_cost": 4.5e-06, - "cache_read_input_token_cost": 3.6e-07, - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3.6e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3.6e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.8e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3.6e-07, - "cache_creation_input_token_cost": 4.5e-06 - }, - "bedrock/us-gov-west-1/anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 3e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-08, - "cache_creation_input_token_cost": 3.75e-07 - }, "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, @@ -14189,34 +13604,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "cerebras/zai-glm-4.6": { - "deprecation_date": "2026-01-20", - "input_cost_per_token": 2.25e-06, - "litellm_provider": "cerebras", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.75e-06, - "source": "https://www.cerebras.ai/pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, - "cerebras/zai-glm-4.7": { - "deprecation_date": "2026-08-17", - "input_cost_per_token": 2.25e-06, - "litellm_provider": "cerebras", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.75e-06, - "source": "https://www.cerebras.ai/pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "cerebras/qwen-3.8-27b": { "input_cost_per_token": 9.9e-07, "litellm_provider": "cerebras", @@ -14242,23 +13629,6 @@ "mode": "chat", "output_cost_per_token": 5e-07 }, - "chatgpt-4o-latest": { - "deprecation_date": "2026-02-17", - "input_cost_per_token": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "gpt-4o-transcribe-diarize": { "input_cost_per_audio_token": 2.5e-06, "input_cost_per_token": 2.5e-06, @@ -14321,131 +13691,6 @@ "prompt_cache_min_tokens": 4096, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, - "claude-3-7-sonnet-20250219": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-02-19", - "input_cost_per_token": 3e-06, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, - "claude-3-haiku-20240307": { - "cache_creation_input_token_cost": 3e-07, - "cache_creation_input_token_cost_above_1hr": 5e-07, - "cache_read_input_token_cost": 3e-08, - "deprecation_date": "2026-04-20", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "claude-3-opus-20240229": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "deprecation_date": "2026-01-05", - "input_cost_per_token": 1.5e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "claude-4-opus-20250514": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_read_input_token_cost": 1.5e-06, - "deprecation_date": "2026-06-15", - "input_cost_per_token": 1.5e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "claude-4-sonnet-20250514": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost": 3e-07, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "deprecation_date": "2026-06-15", - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "litellm_provider": "anthropic", - "max_input_tokens": 1000000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "prompt_cache_min_tokens": 1024 - }, "claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -14516,6 +13761,7 @@ "source": "https://docs.anthropic.com/en/docs/about-claude/pricing" }, "claude-sonnet-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -14555,6 +13801,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-sonnet-4-6": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -14622,92 +13869,6 @@ "input_cost_per_token_batches": 1.5e-06, "output_cost_per_token_batches": 7.5e-06 }, - "claude-opus-4-1": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024, - "deprecation_date": "2026-08-05" - }, - "claude-opus-4-1-20250805": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "deprecation_date": "2026-08-05", - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "claude-opus-4-20250514": { - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "deprecation_date": "2026-06-15", - "litellm_provider": "anthropic", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, "claude-opus-4-5-20251101": { "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -14770,6 +13931,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-6": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14808,6 +13970,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-6-20260205": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14845,6 +14008,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-7": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14884,6 +14048,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-7-20260416": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -14922,6 +14087,7 @@ "prompt_cache_min_tokens": 2048 }, "claude-fable-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -14961,6 +14127,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-fable-5-1": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 2.5e-07, @@ -15000,7 +14167,51 @@ "supports_native_structured_output": true, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, + "claude-opus-5-5": { + "supports_anthropic_compaction": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_native_structured_output": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "provider_specific_entry": { + "fast": 2.0 + }, + "supports_output_config": true, + "supports_speed": true, + "supports_fast_mode": true, + "prompt_cache_min_tokens": 512, + "source": "https://platform.claude.com/docs/en/about-claude/pricing" + }, "claude-opus-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -15043,6 +14254,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-8": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -15084,38 +14296,6 @@ "prompt_cache_min_tokens": 1024, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, - "claude-sonnet-4-20250514": { - "deprecation_date": "2026-06-15", - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "litellm_provider": "anthropic", - "max_input_tokens": 1000000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, "litellm_provider": "cloudflare", @@ -15468,36 +14648,6 @@ "supports_assistant_prefill": true, "supports_tool_choice": true }, - "codex-mini-latest": { - "cache_read_input_token_cost": 3.75e-07, - "deprecation_date": "2026-02-12", - "input_cost_per_token": 1.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 6e-06, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "cohere.command-light-text-v14": { "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", @@ -15508,28 +14658,6 @@ "output_cost_per_token": 6e-07, "supports_tool_choice": true }, - "cohere.command-r-plus-v1:0": { - "deprecation_date": "2026-08-19", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_tool_choice": true - }, - "cohere.command-r-v1:0": { - "deprecation_date": "2026-08-19", - "input_cost_per_token": 5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_tool_choice": true - }, "cohere.command-text-v14": { "input_cost_per_token": 1.5e-06, "litellm_provider": "bedrock", @@ -15619,16 +14747,6 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, - "command": { - "input_cost_per_token": 1e-06, - "litellm_provider": "cohere", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "completion", - "output_cost_per_token": 2e-06, - "deprecation_date": "2025-09-15" - }, "command-a-03-2025": { "input_cost_per_token": 2.5e-06, "litellm_provider": "cohere_chat", @@ -15655,17 +14773,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "command-light": { - "input_cost_per_token": 3e-07, - "litellm_provider": "cohere_chat", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_tool_choice": true, - "deprecation_date": "2025-09-15" - }, "command-nightly": { "input_cost_per_token": 1e-06, "litellm_provider": "cohere", @@ -15675,18 +14782,6 @@ "mode": "completion", "output_cost_per_token": 2e-06 }, - "command-r": { - "input_cost_per_token": 1.5e-07, - "litellm_provider": "cohere_chat", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_function_calling": true, - "supports_tool_choice": true, - "deprecation_date": "2025-09-15" - }, "command-r-08-2024": { "input_cost_per_token": 1.5e-07, "litellm_provider": "cohere_chat", @@ -15698,18 +14793,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "command-r-plus": { - "input_cost_per_token": 2.5e-06, - "litellm_provider": "cohere_chat", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supports_function_calling": true, - "supports_tool_choice": true, - "deprecation_date": "2025-09-15" - }, "command-r-plus-08-2024": { "input_cost_per_token": 2.5e-06, "litellm_provider": "cohere_chat", @@ -15762,26 +14845,6 @@ "supports_vision": true, "source": "https://platform.openai.com/docs/models/computer-use-preview" }, - "dall-e-2": { - "deprecation_date": "2026-05-12", - "input_cost_per_image": 0.02, - "litellm_provider": "openai", - "mode": "image_generation", - "supported_endpoints": [ - "/v1/images/generations", - "/v1/images/edits", - "/v1/images/variations" - ] - }, - "dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_image": 0.04, - "litellm_provider": "openai", - "mode": "image_generation", - "supported_endpoints": [ - "/v1/images/generations" - ] - }, "deepseek-chat": { "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 2.8e-07, @@ -18778,30 +17841,6 @@ "output_vector_size": 1024, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, - "databricks/databricks-claude-3-7-sonnet": { - "cache_creation_input_token_cost": 3.74997e-06, - "cache_read_input_token_cost": 3.0002e-07, - "deprecation_date": "2026-04-12", - "input_cost_per_token": 2.9999900000000002e-06, - "input_dbu_cost_per_token": 4.2857e-05, - "litellm_provider": "databricks", - "max_input_tokens": 200000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, - "output_dbu_cost_per_token": 0.000214286, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_anthropic_thinking_payload": true, - "supports_tool_choice": true - }, "databricks/databricks-claude-fable-5": { "cache_creation_input_token_cost": 1.250004e-05, "cache_read_input_token_cost": 1.00002e-06, @@ -19764,44 +18803,6 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, - "databricks/databricks-gpt-5-1-codex-max": { - "cache_creation_input_token_cost": 1.24999e-06, - "cache_read_input_token_cost": 1.2502e-07, - "deprecation_date": "2026-07-16", - "input_cost_per_token": 1.24999e-06, - "input_dbu_cost_per_token": 1.7857e-05, - "litellm_provider": "databricks", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 9.999990000000002e-06, - "output_dbu_cost_per_token": 0.000142857, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true - }, - "databricks/databricks-gpt-5-1-codex-mini": { - "cache_creation_input_token_cost": 2.4997e-07, - "cache_read_input_token_cost": 2.499e-08, - "deprecation_date": "2026-07-16", - "input_cost_per_token": 2.4997e-07, - "input_dbu_cost_per_token": 3.571e-06, - "litellm_provider": "databricks", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.99997e-06, - "output_dbu_cost_per_token": 2.8571e-05, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true - }, "databricks/databricks-gpt-5-2": { "cache_creation_input_token_cost": 1.75e-06, "cache_read_input_token_cost": 1.75e-07, @@ -19822,25 +18823,6 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, - "databricks/databricks-gpt-5-2-codex": { - "cache_creation_input_token_cost": 1.75e-06, - "cache_read_input_token_cost": 1.75e-07, - "deprecation_date": "2026-07-16", - "input_cost_per_token": 1.75e-06, - "input_dbu_cost_per_token": 2.5e-05, - "litellm_provider": "databricks", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_dbu_cost_per_token": 0.0002, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_prompt_caching": true - }, "databricks/databricks-gpt-5-3-codex": { "cache_creation_input_token_cost": 1.75e-06, "cache_read_input_token_cost": 1.75e-07, @@ -20267,25 +19249,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "databricks/databricks-llama-2-70b-chat": { - "cache_creation_input_token_cost": 5.0001e-07, - "cache_read_input_token_cost": 5.0001e-07, - "deprecation_date": "2024-10-30", - "input_cost_per_token": 5.0001e-07, - "input_dbu_cost_per_token": 7.143e-06, - "litellm_provider": "databricks", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.5000300000000002e-06, - "output_dbu_cost_per_token": 2.1429e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, "databricks/databricks-llama-4-maverick": { "cache_creation_input_token_cost": 5.0001e-07, "cache_read_input_token_cost": 5.0001e-07, @@ -20304,25 +19267,6 @@ "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true }, - "databricks/databricks-meta-llama-3-1-405b-instruct": { - "cache_creation_input_token_cost": 5.00003e-06, - "cache_read_input_token_cost": 5.00003e-06, - "deprecation_date": "2026-02-15", - "input_cost_per_token": 5.00003e-06, - "input_dbu_cost_per_token": 7.1429e-05, - "litellm_provider": "databricks", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, - "output_dbu_cost_per_token": 0.000214286, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, "databricks/databricks-meta-llama-3-1-8b-instruct": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, @@ -20358,82 +19302,6 @@ "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true }, - "databricks/databricks-meta-llama-3-70b-instruct": { - "cache_creation_input_token_cost": 1.00002e-06, - "cache_read_input_token_cost": 1.00002e-06, - "deprecation_date": "2024-07-23", - "input_cost_per_token": 1.00002e-06, - "input_dbu_cost_per_token": 1.4286e-05, - "litellm_provider": "databricks", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 2.9999900000000002e-06, - "output_dbu_cost_per_token": 4.2857e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, - "databricks/databricks-mixtral-8x7b-instruct": { - "cache_creation_input_token_cost": 5.0001e-07, - "cache_read_input_token_cost": 5.0001e-07, - "deprecation_date": "2025-04-30", - "input_cost_per_token": 5.0001e-07, - "input_dbu_cost_per_token": 7.143e-06, - "litellm_provider": "databricks", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.00002e-06, - "output_dbu_cost_per_token": 1.4286e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, - "databricks/databricks-mpt-30b-instruct": { - "cache_creation_input_token_cost": 1.00002e-06, - "cache_read_input_token_cost": 1.00002e-06, - "deprecation_date": "2024-08-30", - "input_cost_per_token": 1.00002e-06, - "input_dbu_cost_per_token": 1.4286e-05, - "litellm_provider": "databricks", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 1.00002e-06, - "output_dbu_cost_per_token": 1.4286e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, - "databricks/databricks-mpt-7b-instruct": { - "cache_creation_input_token_cost": 5.0001e-07, - "cache_read_input_token_cost": 5.0001e-07, - "deprecation_date": "2024-08-30", - "input_cost_per_token": 5.0001e-07, - "input_dbu_cost_per_token": 7.143e-06, - "litellm_provider": "databricks", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." - }, - "mode": "chat", - "output_cost_per_token": 0.0, - "output_dbu_cost_per_token": 0.0, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", - "supports_tool_choice": true - }, "databricks/databricks-qwen35-122b-a10b": { "cache_creation_input_token_cost": 2.2001e-07, "cache_read_input_token_cost": 2.2001e-07, @@ -21498,18 +20366,6 @@ "supports_tool_choice": true, "supports_function_calling": true }, - "deepinfra/google/gemini-2.0-flash-001": { - "deprecation_date": "2026-06-01", - "max_tokens": 1000000, - "max_input_tokens": 1000000, - "max_output_tokens": 1000000, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 4e-07, - "litellm_provider": "deepinfra", - "mode": "chat", - "supports_tool_choice": true, - "supports_function_calling": true - }, "deepinfra/google/gemini-2.5-flash": { "max_tokens": 1000000, "max_input_tokens": 1000000, @@ -22368,15 +21224,6 @@ "/v1/audio/speech" ] }, - "embed-english-light-v2.0": { - "deprecation_date": "2026-04-04", - "input_cost_per_token": 1e-07, - "litellm_provider": "cohere", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, "embed-english-light-v3.0": { "input_cost_per_token": 1e-07, "litellm_provider": "cohere", @@ -22385,15 +21232,6 @@ "mode": "embedding", "output_cost_per_token": 0.0 }, - "embed-english-v2.0": { - "deprecation_date": "2026-04-04", - "input_cost_per_token": 1e-07, - "litellm_provider": "cohere", - "max_input_tokens": 4096, - "max_tokens": 4096, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, "embed-english-v3.0": { "input_cost_per_image": 0.0001, "input_cost_per_token": 1e-07, @@ -22408,15 +21246,6 @@ "supports_embedding_image_input": true, "supports_image_input": true }, - "embed-multilingual-v2.0": { - "deprecation_date": "2026-04-04", - "input_cost_per_token": 1e-07, - "litellm_provider": "cohere", - "max_input_tokens": 768, - "max_tokens": 768, - "mode": "embedding", - "output_cost_per_token": 0.0 - }, "embed-multilingual-v3.0": { "input_cost_per_token": 1e-07, "litellm_provider": "cohere", @@ -22584,23 +21413,6 @@ "cache_read_input_token_cost": 3e-07, "cache_creation_input_token_cost": 3.75e-06 }, - "eu.anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 - }, "eu.anthropic.claude-3-opus-20240229-v1:0": { "input_cost_per_token": 1.5e-05, "litellm_provider": "bedrock", @@ -22616,23 +21428,6 @@ "cache_read_input_token_cost": 1.5e-06, "cache_creation_input_token_cost": 1.875e-05 }, - "eu.anthropic.claude-3-sonnet-20240229-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, "eu.anthropic.claude-opus-4-1-20250805-v1:0": { "cache_creation_input_token_cost": 1.875e-05, "cache_read_input_token_cost": 1.5e-06, @@ -24966,6 +23761,52 @@ "/v1/images/generations" ] }, + "fal_ai/fal-ai/trellis": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.02, + "source": "https://fal.ai/models/fal-ai/trellis", + "metadata": { + "comment": "image-to-3D, returns a GLB mesh; served through the /fal_ai pass-through route" + } + }, + "fal_ai/fal-ai/trellis-2": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.3, + "output_cost_per_image_512": 0.25, + "output_cost_per_image_1024": 0.3, + "output_cost_per_image_1536": 0.35, + "source": "https://fal.ai/models/fal-ai/trellis-2", + "metadata": { + "comment": "image-to-3D, returns a GLB mesh; priced by the request's resolution field (default 1024); served through the /fal_ai pass-through route" + } + }, + "fal_ai/fal-ai/flux-lora-depth": { + "litellm_provider": "fal_ai", + "metadata": { + "notes": "fal bills fal-ai/flux-lora-depth at $0.035 per megapixel, rounding each image up to the nearest megapixel. The per-pixel rate is used when Fal reports the output size, and the flat per-image price prices the default 1 MP output like the sibling flux entries" + }, + "mode": "image_generation", + "output_cost_per_image": 0.035, + "output_cost_per_pixel": 3.337860107421875e-08, + "source": "https://fal.ai/models/fal-ai/flux-lora-depth", + "supported_endpoints": [ + "/v1/images/edits" + ] + }, + "fal_ai/fal-ai/moondream3-preview/query": { + "input_cost_per_token": 4e-07, + "litellm_provider": "fal_ai", + "mode": "chat", + "output_cost_per_token": 3.5e-06, + "source": "https://fal.ai/models/fal-ai/moondream3-preview/query", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_reasoning": true, + "supports_vision": true + }, "featherless_ai/featherless-ai/Qwerky-72B": { "litellm_provider": "featherless_ai", "max_input_tokens": 32768, @@ -25158,26 +23999,6 @@ "supports_tool_choice": true, "supports_vision": false }, - "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro": { - "cache_read_input_token_cost": 6e-07, - "cache_read_input_token_cost_priority": 6e-07, - "deprecation_date": "2026-08-27", - "input_cost_per_token": 1.2e-06, - "input_cost_per_token_priority": 1.2e-06, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_priority": 1.2e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false - }, "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813": { "cache_read_input_token_cost": 4.4e-08, "cache_read_input_token_cost_priority": 5.5e-08, @@ -25565,26 +24386,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "fireworks_ai/accounts/fireworks/models/minimax-m2p7": { - "cache_read_input_token_cost": 6e-08, - "cache_read_input_token_cost_priority": 6e-07, - "deprecation_date": "2026-08-27", - "input_cost_per_token": 3e-07, - "input_cost_per_token_priority": 1.2e-06, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 196608, - "max_output_tokens": 196608, - "max_tokens": 196608, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_priority": 1.2e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false - }, "fireworks_ai/accounts/fireworks/models/minimax-m3": { "cache_read_input_token_cost": 6e-08, "cache_read_input_token_cost_priority": 9e-08, @@ -25672,26 +24473,6 @@ "supports_tool_choice": true, "supports_vision": false }, - "fireworks_ai/deepseek-v4-pro": { - "cache_read_input_token_cost": 6e-07, - "cache_read_input_token_cost_priority": 6e-07, - "deprecation_date": "2026-08-27", - "input_cost_per_token": 1.2e-06, - "input_cost_per_token_priority": 1.2e-06, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_priority": 1.2e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false - }, "fireworks_ai/glm-4p7": { "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 6e-07, @@ -25891,26 +24672,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "fireworks_ai/minimax-m2p7": { - "cache_read_input_token_cost": 6e-08, - "cache_read_input_token_cost_priority": 6e-07, - "deprecation_date": "2026-08-27", - "input_cost_per_token": 3e-07, - "input_cost_per_token_priority": 1.2e-06, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 196608, - "max_output_tokens": 196608, - "max_tokens": 196608, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_priority": 1.2e-06, - "source": "https://api.fireworks.ai/v1/serverless/models", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false - }, "fireworks_ai/minimax-m3": { "cache_read_input_token_cost": 6e-08, "cache_read_input_token_cost_priority": 9e-08, @@ -26088,31 +24849,6 @@ "comment": "Open flagship GLM for long-horizon coding agents and million-token context work", "source": "https://api.friendli.ai/serverless/v1/models" }, - "friendliai/LGAI-EXAONE/K-EXAONE-2.0-750B-A37B": { - "litellm_provider": "friendliai", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "input_cost_per_token": 6e-07, - "output_cost_per_token": 2.4e-06, - "cache_read_input_token_cost": 1.2e-07, - "supports_prompt_caching": true, - "supports_reasoning": true, - "reasoning_effort_levels": [], - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_native_structured_output": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_image_input": false, - "supports_video_input": false, - "mode": "chat", - "comment": "Frontier-scale multilingual language model developed by LG AI Research", - "deprecation_date": "2026-09-06", - "source": "https://api.friendli.ai/serverless/v1/models" - }, "friendliai/deepseek-ai/DeepSeek-V3.2": { "litellm_provider": "friendliai", "max_input_tokens": 163840, @@ -26278,6 +25014,7 @@ }, "ft:gpt-4o-2024-08-06": { "cache_read_input_token_cost": 1.875e-06, + "cache_read_input_token_cost_batches": 9e-07, "input_cost_per_token": 3.75e-06, "input_cost_per_token_batches": 2.225e-06, "litellm_provider": "openai", @@ -26316,6 +25053,7 @@ }, "ft:gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "litellm_provider": "openai", @@ -26336,6 +25074,7 @@ }, "ft:gpt-4.1-2025-04-14": { "cache_read_input_token_cost": 7.5e-07, + "cache_read_input_token_cost_batches": 5e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_batches": 1.5e-06, "litellm_provider": "openai", @@ -26355,6 +25094,7 @@ }, "ft:gpt-4.1-mini-2025-04-14": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_token": 8e-07, "input_cost_per_token_batches": 4e-07, "litellm_provider": "openai", @@ -26374,6 +25114,7 @@ }, "ft:gpt-4.1-nano-2025-04-14": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "deprecation_date": "2026-10-23", "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, @@ -26394,6 +25135,7 @@ }, "ft:o4-mini-2025-04-16": { "cache_read_input_token_cost": 1e-06, + "cache_read_input_token_cost_batches": 5e-07, "deprecation_date": "2026-10-23", "input_cost_per_token": 4e-06, "input_cost_per_token_batches": 2e-06, @@ -26412,160 +25154,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "gemini-2.0-flash": { - "cache_read_input_token_cost": 2.5e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 1e-06, - "input_cost_per_audio_token_batches": 5e-07, - "input_cost_per_character": 3.75e-08, - "input_cost_per_token": 1.5e-07, - "input_cost_per_token_batches": 7.5e-08, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-07, - "output_cost_per_token_batches": 3e-07, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini-2.0-flash-001": { - "cache_read_input_token_cost": 3.75e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 1e-06, - "input_cost_per_token": 1.5e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini-2.0-flash-lite": { - "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7.5e-08, - "input_cost_per_audio_token_batches": 3.75e-08, - "input_cost_per_character": 1.875e-08, - "input_cost_per_token": 7.5e-08, - "input_cost_per_token_batches": 3.75e-08, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "output_cost_per_token_batches": 1.5e-07, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini-2.0-flash-lite-001": { - "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7.5e-08, - "input_cost_per_token": 7.5e-08, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-2.0-flash", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, "gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, "deprecation_date": "2026-10-20", @@ -26677,6 +25265,7 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -26725,7 +25314,8 @@ "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, - "web_search_billing_unit": "per_query" + "web_search_billing_unit": "per_query", + "cache_read_input_token_cost_batches": 1e-07 }, "gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -26769,6 +25359,7 @@ }, "gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -26810,7 +25401,8 @@ "search_context_size_medium": 0.014, "search_context_size_high": 0.014 }, - "web_search_billing_unit": "per_query" + "web_search_billing_unit": "per_query", + "cache_read_input_token_cost_batches": 2.5e-08 }, "gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, @@ -26852,6 +25444,7 @@ }, "gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -26889,7 +25482,8 @@ "supports_response_schema": false, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.25e-08 }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, @@ -26944,6 +25538,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -26998,11 +25593,13 @@ }, "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_audio_token_batches": 2.5e-07 + "input_cost_per_audio_token_batches": 2.5e-07, + "cache_read_input_token_cost_batches": 1.25e-08 }, "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -27055,10 +25652,12 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 1.5e-08 }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -27090,7 +25689,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "cache_read_input_token_cost_batches": 1e-07 }, "gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, @@ -27378,53 +25978,6 @@ }, "gemini_native_audio": true }, - "gemini-2.5-flash-lite-preview-06-17": { - "deprecation_date": "2025-11-18", - "cache_read_input_token_cost": 1e-08, - "input_cost_per_audio_token": 5e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_reasoning_token": 4e-07, - "output_cost_per_token": 4e-07, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "google_maps_grounding_cost_per_query": 0.025, - "supports_image_size": false - }, "gemini-2.5-pro": { "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 1.25e-07, @@ -27483,62 +26036,6 @@ "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 1.8e-05 }, - "gemini-3-pro-preview": { - "deprecation_date": "2026-03-26", - "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, - "input_cost_per_token": 2e-06, - "input_cost_per_token_above_200k_tokens": 4e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "output_cost_per_token_above_200k_tokens": 1.8e-05, - "output_cost_per_token_batches": 6e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_video_input": true, - "supports_vision": true, - "supports_web_search": true, - "supports_native_streaming": true, - "input_cost_per_token_priority": 3.6e-06, - "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, - "output_cost_per_token_priority": 2.16e-05, - "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, - "cache_read_input_token_cost_priority": 3.6e-07, - "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query" - }, "gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, @@ -27769,6 +26266,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_token": 1.5e-06, "input_cost_per_audio_token": 1.5e-06, "litellm_provider": "vertex_ai", @@ -27823,11 +26321,13 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_read_input_token_cost_batches": 7.5e-08 }, "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -27880,11 +26380,13 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "vertex_ai/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -27938,11 +26440,13 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "vertex_ai/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -27996,7 +26500,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "vertex_ai/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, @@ -28189,53 +26694,9 @@ "supports_url_context": true, "supports_vision": true }, - "gemini/gemini-robotics-er-1.5-preview": { - "cache_read_input_token_cost": 0, - "deprecation_date": "2026-04-30", - "input_cost_per_token": 3e-07, - "input_cost_per_audio_token": 1e-06, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "output_cost_per_reasoning_token": 2.5e-06, - "source": "https://ai.google.dev/gemini-api/docs/models#gemini-robotics-er-1-5-preview", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions" - ], - "supported_modalities": [ - "text", - "image", - "video", - "audio" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": false, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000, - "rpm": 10, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, "gemini/gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_audio_token": 2e-06, "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, @@ -28283,53 +26744,6 @@ "supports_web_search": true, "web_search_billing_unit": "per_query" }, - "gemini/gemini-robotics-er-1.6-preview": { - "deprecation_date": "2026-08-31", - "input_cost_per_audio_token": 2e-06, - "input_cost_per_token": 1e-06, - "litellm_provider": "gemini", - "max_input_tokens": 131072, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_reasoning_token": 5e-06, - "output_cost_per_token": 5e-06, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-robotics-er", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_video_input": true, - "supports_vision": true, - "supports_web_search": true, - "web_search_billing_unit": "per_query" - }, "gemini-2.5-computer-use-preview-10-2025": { "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, @@ -28466,27 +26880,6 @@ "source": "https://ai.google.dev/gemini-api/docs/embeddings#model-versions", "tpm": 10000000 }, - "gemini/gemini-embedding-2-preview": { - "deprecation_date": "2026-08-10", - "input_cost_per_audio_token": 6.5e-06, - "input_cost_per_audio_token_batches": 3.25e-06, - "input_cost_per_image_token": 4.5e-07, - "input_cost_per_image_token_batches": 2.25e-07, - "input_cost_per_token": 2e-07, - "input_cost_per_token_batches": 1e-07, - "input_cost_per_video_token": 1.2e-05, - "input_cost_per_video_token_batches": 6e-06, - "litellm_provider": "gemini", - "max_input_tokens": 8192, - "max_tokens": 8192, - "mode": "embedding", - "output_cost_per_token": 0, - "output_vector_size": 3072, - "rpm": 10000, - "source": "https://ai.google.dev/gemini-api/docs/pricing", - "supports_multimodal": true, - "tpm": 10000000 - }, "gemini/gemini-embedding-2": { "input_cost_per_audio_token": 6.5e-06, "input_cost_per_audio_token_batches": 3.25e-06, @@ -28509,138 +26902,10 @@ "supports_vision": true, "tpm": 10000000 }, - "gemini/gemini-1.5-flash": { - "deprecation_date": "2025-09-29", - "input_cost_per_token": 7.5e-08, - "input_cost_per_token_above_128k_tokens": 1.5e-07, - "litellm_provider": "gemini", - "max_input_tokens": 8192, - "max_tokens": 8192, - "mode": "embedding", - "output_cost_per_token": 0, - "output_vector_size": 3072, - "rpm": 10000, - "source": "https://ai.google.dev/gemini-api/docs/embeddings#multimodal", - "supports_multimodal": true, - "tpm": 10000000 - }, - "gemini/gemini-2.0-flash": { - "cache_read_input_token_cost": 2.5e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4e-07, - "rpm": 10000, - "source": "https://ai.google.dev/pricing#2_0flash", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 10000000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini/gemini-2.0-flash-001": { - "cache_read_input_token_cost": 2.5e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4e-07, - "rpm": 10000, - "source": "https://ai.google.dev/pricing#2_0flash", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 10000000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, - "gemini/gemini-2.0-flash-lite": { - "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7.5e-08, - "input_cost_per_token": 7.5e-08, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "rpm": 4000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.0-flash-lite", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 4000000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, "gemini/gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_audio_token": 1e-06, @@ -28799,50 +27064,6 @@ "web_search_billing_unit": "per_query", "supports_reasoning": false }, - "gemini/gemini-3-pro-image-preview": { - "deprecation_date": "2026-06-25", - "input_cost_per_image": 0.0011, - "input_cost_per_token": 2e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "gemini", - "max_input_tokens": 65536, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "image_generation", - "output_cost_per_image": 0.134, - "output_cost_per_image_token": 0.00012, - "output_cost_per_token": 1.2e-05, - "rpm": 1000, - "tpm": 4000000, - "output_cost_per_token_batches": 6e-06, - "source": "https://ai.google.dev/gemini-api/docs/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_prompt_caching": true, - "supports_reasoning": false, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query" - }, "gemini/nano-banana-pro-preview": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -28928,49 +27149,6 @@ }, "web_search_billing_unit": "per_query" }, - "gemini/gemini-3.1-flash-image-preview": { - "deprecation_date": "2026-06-25", - "input_cost_per_token": 5e-07, - "input_cost_per_token_batches": 2.5e-07, - "litellm_provider": "gemini", - "max_input_tokens": 65536, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "image_generation", - "output_cost_per_image": 0.045, - "output_cost_per_image_token": 6e-05, - "output_cost_per_token": 3e-06, - "output_cost_per_token_batches": 1.5e-06, - "rpm": 1000, - "tpm": 4000000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-image-preview", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_prompt_caching": true, - "supports_reasoning": false, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_vision": true, - "supports_web_search": true, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query" - }, "gemini/gemini-3.1-flash-lite-image": { "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -29052,6 +27230,7 @@ "gemini/gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, "input_cost_per_audio_token": 3e-07, @@ -29108,104 +27287,6 @@ "supports_audio_input": true, "supports_image_size": false }, - "gemini/gemini-2.5-flash-lite-preview-09-2025": { - "cache_read_input_token_cost": 1e-08, - "deprecation_date": "2026-03-31", - "input_cost_per_audio_token": 3e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_reasoning_token": 4e-07, - "output_cost_per_token": 4e-07, - "rpm": 15, - "source": "https://developers.googleblog.com/en/continuing-to-bring-you-our-latest-models-with-an-improved-gemini-2-5-flash-and-flash-lite-release/", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "google_maps_grounding_cost_per_query": 0.025, - "supports_image_size": false - }, - "gemini/gemini-2.5-flash-preview-09-2025": { - "cache_read_input_token_cost": 3e-08, - "deprecation_date": "2026-02-17", - "input_cost_per_audio_token": 1e-06, - "input_cost_per_token": 3e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_reasoning_token": 2.5e-06, - "output_cost_per_token": 2.5e-06, - "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "google_maps_grounding_cost_per_query": 0.025, - "supports_image_size": false - }, "gemini/gemini-flash-latest": { "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, @@ -29323,55 +27404,6 @@ "supports_video_input": true, "web_search_billing_unit": "per_query" }, - "gemini/gemini-2.5-flash-lite-preview-06-17": { - "deprecation_date": "2025-11-18", - "cache_read_input_token_cost": 1e-08, - "input_cost_per_audio_token": 5e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_reasoning_token": 4e-07, - "output_cost_per_token": 4e-07, - "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 250000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - }, - "google_maps_grounding_cost_per_query": 0.025, - "supports_image_size": false - }, "gemini/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, @@ -29397,6 +27429,7 @@ "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, "cache_read_input_token_cost_flex": 1.25e-07, "cache_read_input_token_cost_priority": 2.25e-07, "input_cost_per_token": 1.25e-06, @@ -29481,117 +27514,10 @@ "supports_vision": true, "tpm": 800000 }, - "gemini/gemini-3-pro-preview": { - "deprecation_date": "2026-03-09", - "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "input_cost_per_token": 2e-06, - "input_cost_per_token_above_200k_tokens": 4e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, - "max_tokens": 65535, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "output_cost_per_token_above_200k_tokens": 1.8e-05, - "output_cost_per_token_batches": 6e-06, - "rpm": 2000, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_video_input": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 800000, - "input_cost_per_token_priority": 3.6e-06, - "input_cost_per_token_above_200k_tokens_priority": 7.2e-06, - "output_cost_per_token_priority": 2.16e-05, - "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, - "cache_read_input_token_cost_priority": 3.6e-07, - "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query" - }, - "gemini/gemini-3.1-flash-lite-preview": { - "cache_read_input_token_cost": 2.5e-08, - "deprecation_date": "2026-05-25", - "input_cost_per_audio_token": 5e-07, - "input_cost_per_token": 2.5e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_reasoning_token": 1.5e-06, - "output_cost_per_token": 1.5e-06, - "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/models", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/completions", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_audio_output": false, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_url_context": true, - "supports_video_input": true, - "supports_vision": true, - "supports_web_search": true, - "supports_native_streaming": true, - "tpm": 250000, - "search_context_cost_per_query": { - "search_context_size_low": 0.014, - "search_context_size_medium": 0.014, - "search_context_size_high": 0.014 - }, - "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 - }, "gemini/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-05-07", @@ -29653,6 +27579,7 @@ }, "gemini/gemini-3.5-flash-lite": { "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 2e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, "input_cost_per_token": 3e-07, @@ -29712,6 +27639,7 @@ "gemini/gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 5e-07, @@ -29773,6 +27701,7 @@ "gemini/gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "gemini", @@ -29834,6 +27763,7 @@ "gemini/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -29893,6 +27823,7 @@ "gemini/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -29953,6 +27884,7 @@ "gemini/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30095,6 +28027,7 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 2e-07, "cache_read_input_token_cost_flex": 2e-07, "search_context_cost_per_query": { "search_context_size_low": 0.014, @@ -30157,6 +28090,7 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 2e-07, "cache_read_input_token_cost_flex": 2e-07, "search_context_cost_per_query": { "search_context_size_low": 0.014, @@ -30261,6 +28195,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", @@ -30315,11 +28250,13 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_read_input_token_cost_batches": 7.5e-08 }, "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30372,11 +28309,13 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30430,11 +28369,13 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30488,7 +28429,8 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 3.75e-08 }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, @@ -30672,34 +28614,6 @@ "output_cost_per_image": 0.04, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, - "gemini/imagen-3.0-generate-002": { - "deprecation_date": "2025-11-10", - "litellm_provider": "gemini", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "gemini/imagen-4.0-fast-generate-001": { - "deprecation_date": "2026-08-17", - "litellm_provider": "gemini", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "gemini/imagen-4.0-generate-001": { - "deprecation_date": "2026-08-17", - "litellm_provider": "gemini", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "gemini/imagen-4.0-ultra-generate-001": { - "deprecation_date": "2026-08-17", - "litellm_provider": "gemini", - "mode": "image_generation", - "output_cost_per_image": 0.06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, "gemini/learnlm-1.5-pro-experimental": { "input_cost_per_audio_per_second": 0, "input_cost_per_audio_per_second_above_128k_tokens": 0, @@ -30778,21 +28692,6 @@ "supports_web_search": false, "output_cost_per_image": 0.08 }, - "gemini/veo-2.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "gemini", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "video_generation", - "output_cost_per_second": 0.35, - "source": "https://ai.google.dev/gemini-api/docs/video", - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "video" - ] - }, "gemini/veo-3.1-fast-generate-preview": { "litellm_provider": "gemini", "max_input_tokens": 1024, @@ -31652,10 +29551,21 @@ "output_cost_per_token": 3.15e-06 }, "baseten/zai-org/GLM-4.7": { + "cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 6e-07, "litellm_provider": "baseten", + "max_input_tokens": 200000, + "max_output_tokens": 200000, + "max_tokens": 200000, "mode": "chat", - "output_cost_per_token": 2.2e-06 + "output_cost_per_token": 2.2e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": false, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false }, "baseten/zai-org/GLM-4.6": { "input_cost_per_token": 6e-07, @@ -31682,10 +29592,21 @@ "output_cost_per_token": 2.5e-06 }, "baseten/openai/gpt-oss-120b": { + "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-07, "litellm_provider": "baseten", + "max_input_tokens": 128072, + "max_output_tokens": 128072, + "max_tokens": 128072, "mode": "chat", - "output_cost_per_token": 5e-07 + "output_cost_per_token": 5e-07, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false }, "baseten/deepseek-ai/DeepSeek-V3.1": { "input_cost_per_token": 5e-07, @@ -31980,33 +29901,6 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "gpt-4-0125-preview": { - "deprecation_date": "2026-03-26", - "input_cost_per_token": 1e-05, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 3e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, - "gpt-4-0314": { - "deprecation_date": "2026-03-26", - "input_cost_per_token": 3e-05, - "litellm_provider": "openai", - "max_input_tokens": 8192, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 6e-05, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4-0613": { "deprecation_date": "2026-10-23", "input_cost_per_token": 3e-05, @@ -32076,22 +29970,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "gpt-4-turbo-preview": { - "deprecation_date": "2026-03-26", - "input_cost_per_token": 1e-05, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 3e-05, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4.1": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_priority": 8.75e-07, @@ -32434,24 +30312,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "gpt-4o-audio-preview": { - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 2.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_token": 1e-05, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4o-audio-preview-2024-12-17": { "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 4e-05, @@ -32636,44 +30496,6 @@ "supports_tool_choice": true, "supports_vision": false }, - "gpt-audio-mini-2025-10-06": { - "deprecation_date": "2026-07-23", - "input_cost_per_audio_token": 1e-05, - "input_cost_per_token": 6e-07, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 2.4e-06, - "source": "https://developers.openai.com/api/docs/pricing", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses", - "/v1/realtime", - "/v1/batch" - ], - "supported_modalities": [ - "text", - "audio" - ], - "supported_output_modalities": [ - "text", - "audio" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": false, - "supports_reasoning": false, - "supports_response_schema": false, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": false - }, "gpt-audio-mini-2025-12-15": { "input_cost_per_audio_token": 1e-05, "input_cost_per_token": 6e-07, @@ -32770,24 +30592,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "gpt-4o-mini-audio-preview": { - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 1e-05, - "input_cost_per_token": 1.5e-07, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 6e-07, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4o-mini-audio-preview-2024-12-17": { "deprecation_date": "2027-01-20", "input_cost_per_audio_token": 1e-05, @@ -32806,26 +30610,6 @@ "supports_system_messages": true, "supports_tool_choice": true }, - "gpt-4o-mini-realtime-preview": { - "cache_creation_input_audio_token_cost": 3e-07, - "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 1e-05, - "input_cost_per_token": 6e-07, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 2.4e-06, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4o-mini-realtime-preview-2024-12-17": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_token_cost": 3e-07, @@ -32872,32 +30656,6 @@ "supports_vision": true, "supports_web_search": true }, - "gpt-4o-mini-search-preview-2025-03-11": { - "cache_read_input_token_cost": 7.5e-08, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.5e-07, - "input_cost_per_token_batches": 7.5e-08, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 6e-07, - "output_cost_per_token_batches": 3e-07, - "search_context_cost_per_query": { - "search_context_size_high": 0.025, - "search_context_size_low": 0.025, - "search_context_size_medium": 0.025 - }, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "gpt-4o-mini-transcribe": { "input_cost_per_audio_token": 1.25e-06, "input_cost_per_token": 1.25e-06, @@ -32932,63 +30690,6 @@ "audio" ] }, - "gpt-4o-realtime-preview": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_token": 2e-05, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, - "gpt-4o-realtime-preview-2024-12-17": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_token": 2e-05, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, - "gpt-4o-realtime-preview-2025-06-03": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-05-07", - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_token": 2e-05, - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-4o-search-preview": { "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, @@ -33015,32 +30716,6 @@ "supports_vision": true, "supports_web_search": true }, - "gpt-4o-search-preview-2025-03-11": { - "cache_read_input_token_cost": 1.25e-06, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 2.5e-06, - "input_cost_per_token_batches": 1.25e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "output_cost_per_token_batches": 5e-06, - "search_context_cost_per_query": { - "search_context_size_high": 0.025, - "search_context_size_low": 0.025, - "search_context_size_medium": 0.025 - }, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "gpt-4o-transcribe": { "input_cost_per_audio_token": 2.5e-06, "input_cost_per_token": 2.5e-06, @@ -33058,6 +30733,7 @@ }, "gpt-image-1.5": { "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", "input_cost_per_token": 5e-06, "litellm_provider": "openai", @@ -33077,6 +30753,7 @@ }, "gpt-image-1.5-2025-12-16": { "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", "input_cost_per_token": 5e-06, "litellm_provider": "openai", @@ -33096,6 +30773,7 @@ }, "gpt-image-2": { "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_batches": 6.25e-07, "input_cost_per_token": 5e-06, "litellm_provider": "openai", "mode": "image_generation", @@ -33547,6 +31225,7 @@ }, "gpt-5": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -33597,6 +31276,7 @@ }, "gpt-5.1": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -33648,6 +31328,7 @@ }, "gpt-5.1-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -33697,54 +31378,9 @@ "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": false }, - "gpt-5.1-chat-latest": { - "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_priority": 2.5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "input_cost_per_token_priority": 2.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "output_cost_per_token_priority": 2e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text", - "image" - ], - "supports_function_calling": false, - "supports_native_streaming": true, - "supports_parallel_function_calling": false, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": false, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": true, - "default_reasoning_effort": "none", - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, "gpt-5.2": { "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_batches": 8.75e-08, "cache_read_input_token_cost_flex": 8.75e-08, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, @@ -33797,6 +31433,7 @@ }, "gpt-5.2-2025-12-11": { "cache_read_input_token_cost": 1.75e-07, + "cache_read_input_token_cost_batches": 8.75e-08, "cache_read_input_token_cost_flex": 8.75e-08, "cache_read_input_token_cost_priority": 3.5e-07, "input_cost_per_token": 1.75e-06, @@ -33847,94 +31484,6 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, - "gpt-5.2-chat-latest": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-08-10", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.3-chat-latest": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-08-10", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, "gpt-5.2-pro": { "input_cost_per_token": 2.1e-05, "input_cost_per_token_batches": 1.05e-05, @@ -34030,6 +31579,10 @@ "cache_read_input_token_cost_above_272k_tokens": 2e-06, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06, + "cache_read_input_token_cost_batches": 5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 1e-06, + "cache_creation_input_token_cost_batches": 6.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-05, "cache_read_input_token_cost_flex": 5e-07, "cache_read_input_token_cost_priority": 2e-06, "input_cost_per_token": 1e-05, @@ -34037,6 +31590,7 @@ "input_cost_per_token_above_272k_tokens_flex": 1e-05, "input_cost_per_token_above_272k_tokens_priority": 4e-05, "input_cost_per_token_batches": 5e-06, + "input_cost_per_token_above_272k_tokens_batches": 1e-05, "input_cost_per_token_flex": 5e-06, "input_cost_per_token_priority": 2e-05, "litellm_provider": "openai", @@ -34049,6 +31603,7 @@ "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, "output_cost_per_token_above_272k_tokens_priority": 0.00015, "output_cost_per_token_batches": 2.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 3.75e-05, "output_cost_per_token_flex": 2.5e-05, "output_cost_per_token_priority": 0.0001, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34089,6 +31644,158 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-6-sol": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, + "cache_creation_input_token_cost_flex": 1.25e-06, + "cache_creation_input_token_cost_priority": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_read_input_token_cost_batches": 1e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2e-07, + "cache_creation_input_token_cost_batches": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, + "cache_read_input_token_cost_flex": 1e-07, + "cache_read_input_token_cost_priority": 4e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "input_cost_per_token_above_272k_tokens_flex": 2e-06, + "input_cost_per_token_above_272k_tokens_priority": 8e-06, + "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_above_272k_tokens_batches": 2e-06, + "input_cost_per_token_flex": 1e-06, + "input_cost_per_token_priority": 4e-06, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_272k_tokens": 1.5e-05, + "output_cost_per_token_above_272k_tokens_flex": 7.5e-06, + "output_cost_per_token_above_272k_tokens_priority": 3e-05, + "output_cost_per_token_batches": 5e-06, + "output_cost_per_token_above_272k_tokens_batches": 7.5e-06, + "output_cost_per_token_flex": 5e-06, + "output_cost_per_token_priority": 2e-05, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, + "gpt-6-luna": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07, + "cache_creation_input_token_cost_flex": 6.25e-08, + "cache_creation_input_token_cost_priority": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 4e-08, + "cache_read_input_token_cost_batches": 5e-09, + "cache_read_input_token_cost_above_272k_tokens_batches": 1e-08, + "cache_creation_input_token_cost_batches": 6.25e-08, + "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-07, + "cache_read_input_token_cost_flex": 5e-09, + "cache_read_input_token_cost_priority": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "input_cost_per_token_above_272k_tokens_flex": 1e-07, + "input_cost_per_token_above_272k_tokens_priority": 4e-07, + "input_cost_per_token_batches": 5e-08, + "input_cost_per_token_above_272k_tokens_batches": 1e-07, + "input_cost_per_token_flex": 5e-08, + "input_cost_per_token_priority": 2e-07, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_above_272k_tokens": 7.5e-07, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-07, + "output_cost_per_token_above_272k_tokens_priority": 1.5e-06, + "output_cost_per_token_batches": 2.5e-07, + "output_cost_per_token_above_272k_tokens_batches": 3.75e-07, + "output_cost_per_token_flex": 2.5e-07, + "output_cost_per_token_priority": 1e-06, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, @@ -34100,6 +31807,10 @@ "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_read_input_token_cost_batches": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 4e-07, + "cache_creation_input_token_cost_batches": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 5e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, @@ -34107,6 +31818,7 @@ "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 4e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", @@ -34119,6 +31831,7 @@ "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.5e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34167,6 +31880,10 @@ "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_read_input_token_cost_batches": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 4e-07, + "cache_creation_input_token_cost_batches": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 5e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, @@ -34174,6 +31891,7 @@ "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 4e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", @@ -34186,6 +31904,7 @@ "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.5e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34236,6 +31955,10 @@ "cache_read_input_token_cost_above_272k_tokens": 4e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_read_input_token_cost_batches": 1e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2e-07, + "cache_creation_input_token_cost_batches": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 4e-07, "input_cost_per_token": 2e-06, @@ -34243,6 +31966,7 @@ "input_cost_per_token_above_272k_tokens_flex": 2e-06, "input_cost_per_token_above_272k_tokens_priority": 8e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_above_272k_tokens_batches": 2e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, "litellm_provider": "openai", @@ -34255,6 +31979,7 @@ "output_cost_per_token_above_272k_tokens_flex": 9e-06, "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, "output_cost_per_token_batches": 6e-06, + "output_cost_per_token_above_272k_tokens_batches": 9e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34304,6 +32029,10 @@ "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-08, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, + "cache_read_input_token_cost_batches": 1e-08, + "cache_read_input_token_cost_above_272k_tokens_batches": 2e-08, + "cache_creation_input_token_cost_batches": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -34311,6 +32040,7 @@ "input_cost_per_token_above_272k_tokens_flex": 2e-07, "input_cost_per_token_above_272k_tokens_priority": 8e-07, "input_cost_per_token_batches": 1e-07, + "input_cost_per_token_above_272k_tokens_batches": 2e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_priority": 4e-07, "litellm_provider": "openai", @@ -34323,6 +32053,7 @@ "output_cost_per_token_above_272k_tokens_flex": 9e-07, "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_above_272k_tokens_batches": 9e-07, "output_cost_per_token_flex": 6e-07, "output_cost_per_token_priority": 2.4e-06, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34593,12 +32324,15 @@ "gpt-5.5": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost_batches": 2.5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34609,6 +32343,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34651,12 +32386,15 @@ "gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost_batches": 2.5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34667,6 +32405,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34711,6 +32450,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -34720,6 +32460,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -34760,6 +32501,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -34769,6 +32511,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -34807,12 +32550,15 @@ "gpt-5.4": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_batches": 1.3e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_priority": 5e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34823,6 +32569,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_priority": 3e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34860,12 +32607,15 @@ "gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_batches": 1.3e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_priority": 5e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34876,6 +32626,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_priority": 3e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34915,6 +32666,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -34924,6 +32676,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -34965,6 +32718,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -34974,6 +32728,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -35012,6 +32767,7 @@ }, "gpt-5.4-mini": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -35065,6 +32821,7 @@ }, "gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.5e-07, "input_cost_per_token": 7.5e-07, @@ -35118,6 +32875,7 @@ }, "gpt-5.4-nano": { "cache_read_input_token_cost": 2e-08, + "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "input_cost_per_token": 2e-07, "input_cost_per_token_flex": 1e-07, @@ -35168,6 +32926,7 @@ }, "gpt-5.4-nano-2026-03-17": { "cache_read_input_token_cost": 2e-08, + "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "input_cost_per_token": 2e-07, "input_cost_per_token_flex": 1e-07, @@ -35303,6 +33062,7 @@ }, "gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", @@ -35350,7 +33110,8 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_batches": 6.25e-08 }, "gpt-5-chat": { "cache_read_input_token_cost": 1.25e-07, @@ -35387,251 +33148,6 @@ "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": true }, - "gpt-5-chat-latest": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": false, - "supports_native_streaming": true, - "supports_parallel_function_calling": false, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": false, - "supports_vision": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5-codex": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 1e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.1-codex": { - "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_priority": 2.5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "input_cost_per_token_priority": 2.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 1e-05, - "output_cost_per_token_priority": 2e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.1-codex-max": { - "cache_read_input_token_cost": 1.25e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 1e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.1-codex-mini": { - "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_priority": 4.5e-08, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 2.5e-07, - "input_cost_per_token_priority": 4.5e-07, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 2e-06, - "output_cost_per_token_priority": 3.6e-06, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true - }, - "gpt-5.2-codex": { - "cache_read_input_token_cost": 1.75e-07, - "cache_read_input_token_cost_priority": 3.5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_priority": 2.8e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": true, - "supports_minimal_reasoning_effort": true - }, "gpt-5.3-codex": { "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, @@ -35677,6 +33193,7 @@ }, "gpt-5-mini": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_token": 2.5e-07, @@ -35727,6 +33244,7 @@ }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", @@ -35774,10 +33292,12 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_batches": 1.25e-08 }, "gpt-5-nano": { "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, @@ -35826,6 +33346,7 @@ }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-08, @@ -35871,10 +33392,12 @@ "supports_web_search": true, "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_minimal_reasoning_effort": true, + "cache_read_input_token_cost_batches": 2.5e-09 }, "gpt-image-1": { "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-10-23", "input_cost_per_image_token": 1e-05, "input_cost_per_image_token_batches": 5e-06, @@ -35891,6 +33414,7 @@ }, "gpt-image-1-mini": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "deprecation_date": "2026-12-01", "input_cost_per_image_token": 2.5e-06, "input_cost_per_image_token_batches": 1.25e-06, @@ -36466,44 +33990,14 @@ "supports_response_schema": true, "supports_vision": true }, - "groq/llama-3.1-8b-instant": { - "deprecation_date": "2026-08-16", - "input_cost_per_token": 5e-08, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 8e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/llama-3.3-70b-versatile": { - "deprecation_date": "2026-08-16", - "input_cost_per_token": 5.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 7.9e-07, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, - "groq/gemma-7b-it": { - "deprecation_date": "2024-12-18", - "input_cost_per_token": 5e-08, + "groq/llama-guard-3-8b": { + "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 8192, - "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 8e-08, - "supports_function_calling": true, - "supports_response_schema": false, - "supports_tool_choice": true + "output_cost_per_token": 2e-07, + "source": "https://console.groq.com/docs/model/llama-guard-3-8b" }, "groq/meta-llama/llama-prompt-guard-2-22m": { "input_cost_per_token": 3e-08, @@ -36525,58 +34019,6 @@ "output_cost_per_token": 4e-08, "source": "https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m" }, - "groq/meta-llama/llama-guard-4-12b": { - "deprecation_date": "2026-03-05", - "input_cost_per_token": 2e-07, - "litellm_provider": "groq", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-07 - }, - "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { - "deprecation_date": "2026-03-09", - "input_cost_per_token": 2e-07, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/meta-llama/llama-4-scout-17b-16e-instruct": { - "deprecation_date": "2026-07-17", - "input_cost_per_token": 1.1e-07, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3.4e-07, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/moonshotai/kimi-k2-instruct-0905": { - "deprecation_date": "2026-04-15", - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3e-06, - "cache_read_input_token_cost": 5e-07, - "litellm_provider": "groq", - "max_input_tokens": 262144, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "groq/openai/gpt-oss-120b": { "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.5e-07, @@ -36657,45 +34099,6 @@ "mode": "audio_speech", "source": "https://console.groq.com/docs/models" }, - "groq/playai-tts": { - "deprecation_date": "2025-12-31", - "input_cost_per_character": 5e-05, - "litellm_provider": "groq", - "max_input_tokens": 10000, - "max_output_tokens": 10000, - "max_tokens": 10000, - "mode": "audio_speech" - }, - "groq/qwen/qwen3.6-27b": { - "input_cost_per_token": 6e-07, - "litellm_provider": "groq", - "max_input_tokens": 131072, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://console.groq.com/docs/model/qwen/qwen3.6-27b", - "deprecation_date": "2026-09-14", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_vision": true - }, - "groq/qwen/qwen3-32b": { - "deprecation_date": "2026-07-17", - "input_cost_per_token": 2.9e-07, - "litellm_provider": "groq", - "max_input_tokens": 131000, - "max_output_tokens": 131000, - "max_tokens": 131000, - "mode": "chat", - "output_cost_per_token": 5.9e-07, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true - }, "groq/whisper-large-v3": { "input_cost_per_second": 3.083e-05, "litellm_provider": "groq", @@ -36708,27 +34111,6 @@ "mode": "audio_transcription", "output_cost_per_second": 0.0 }, - "hd/1024-x-1024/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 7.629e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, - "hd/1024-x-1792/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 6.539e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, - "hd/1792-x-1024/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 6.539e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "heroku/claude-3-5-haiku": { "litellm_provider": "heroku", "max_tokens": 8192, @@ -38555,19 +35937,6 @@ "supports_system_messages": true, "supports_native_structured_output": true }, - "mistral/codestral-2405": { - "deprecation_date": "2025-06-16", - "input_cost_per_token": 1e-06, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 3e-06, - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/codestral-2508": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, @@ -38611,51 +35980,6 @@ "supports_assistant_prefill": true, "supports_tool_choice": true }, - "mistral/devstral-medium-2507": { - "deprecation_date": "2026-05-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://mistral.ai/news/devstral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/devstral-small-2505": { - "deprecation_date": "2025-11-30", - "input_cost_per_token": 1e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://mistral.ai/news/devstral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/devstral-small-2507": { - "deprecation_date": "2026-05-31", - "input_cost_per_token": 1e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://mistral.ai/news/devstral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/devstral-small-latest": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, @@ -38671,21 +35995,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/labs-devstral-small-2512": { - "deprecation_date": "2026-03-31", - "input_cost_per_token": 1e-07, - "litellm_provider": "mistral", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://docs.mistral.ai/models/devstral-small-2-25-12", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/devstral-latest": { "cache_read_input_token_cost": 4e-08, "input_cost_per_token": 4e-07, @@ -38716,21 +36025,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/devstral-2512": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://mistral.ai/news/devstral-2-vibe-cli", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/ministral-14b-2512": { "input_cost_per_token": 2e-07, "litellm_provider": "mistral", @@ -39006,54 +36300,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/magistral-medium-2506": { - "deprecation_date": "2025-11-30", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/magistral-medium-2509": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/magistral-medium-1-2-2509": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://mistral.ai/news/magistral", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/mistral-ocr-latest": { "litellm_provider": "mistral", "ocr_cost_per_page": 0.004, @@ -39093,20 +36339,6 @@ "/v1/batch" ] }, - "mistral/mistral-ocr-2505-completion": { - "deprecation_date": "2026-05-31", - "litellm_provider": "mistral", - "ocr_cost_per_page": 0.001, - "ocr_cost_per_page_batches": 0.0005, - "annotation_cost_per_page": 0.003, - "annotation_cost_per_page_batches": 0.0015, - "mode": "ocr", - "supported_endpoints": [ - "/v1/ocr", - "/v1/batch" - ], - "source": "https://mistral.ai/pricing#api-pricing" - }, "mistral/mistral-ocr-2512": { "litellm_provider": "mistral", "ocr_cost_per_page": 0.002, @@ -39137,22 +36369,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/magistral-small-2506": { - "deprecation_date": "2025-11-30", - "input_cost_per_token": 5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://mistral.ai/pricing#api-pricing", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/magistral-small-latest": { "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, @@ -39170,22 +36386,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/magistral-small-1-2-2509": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 40000, - "max_output_tokens": 40000, - "max_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://mistral.ai/pricing#api-pricing", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/mistral-embed": { "input_cost_per_token": 1e-07, "litellm_provider": "mistral", @@ -39209,48 +36409,6 @@ "max_tokens": 8192, "mode": "embedding" }, - "mistral/mistral-large-2402": { - "deprecation_date": "2025-06-16", - "input_cost_per_token": 4e-06, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 1.2e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/mistral-large-2407": { - "deprecation_date": "2025-03-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 9e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/mistral-large-2411": { - "deprecation_date": "2026-05-31", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 6e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/mistral-large-latest": { "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 5e-07, @@ -39320,49 +36478,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/mistral-medium-2312": { - "deprecation_date": "2025-06-16", - "input_cost_per_token": 2.7e-06, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 8.1e-06, - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/mistral-medium-2505": { - "deprecation_date": "2026-08-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 2e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/mistral-medium-2508": { - "deprecation_date": "2026-08-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://mistral.ai/news/mistral-medium-3", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "mistral/mistral-medium-2604": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, @@ -39405,22 +36520,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/mistral-medium-3-1-2508": { - "deprecation_date": "2026-08-31", - "input_cost_per_token": 4e-07, - "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://mistral.ai/news/mistral-medium-3", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "mistral/mistral-medium-3-5": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, @@ -39477,22 +36576,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "mistral/mistral-small-3-2-2506": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 6e-08, - "litellm_provider": "mistral", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "source": "https://mistral.ai/pricing", - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "mistral/ministral-3-3b-2512": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, @@ -39586,32 +36669,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/open-codestral-mamba": { - "deprecation_date": "2025-06-06", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2.5e-07, - "source": "https://mistral.ai/technology/", - "supports_assistant_prefill": true, - "supports_tool_choice": true - }, - "mistral/open-mistral-7b": { - "deprecation_date": "2025-03-30", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 2.5e-07, - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "mistral/open-mistral-nemo": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, @@ -39626,78 +36683,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "mistral/open-mistral-nemo-2407": { - "deprecation_date": "2026-07-31", - "input_cost_per_token": 3e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://mistral.ai/technology/", - "supports_assistant_prefill": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/open-mixtral-8x22b": { - "deprecation_date": "2025-03-30", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 65336, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 6e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/open-mixtral-8x7b": { - "deprecation_date": "2025-03-30", - "input_cost_per_token": 7e-07, - "litellm_provider": "mistral", - "max_input_tokens": 32000, - "max_output_tokens": 8191, - "max_tokens": 8191, - "mode": "chat", - "output_cost_per_token": 7e-07, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "mistral/pixtral-12b-2409": { - "deprecation_date": "2025-12-31", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 1.5e-07, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "mistral/pixtral-large-2411": { - "deprecation_date": "2026-05-31", - "input_cost_per_token": 2e-06, - "litellm_provider": "mistral", - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 6e-06, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "mistral/pixtral-large-latest": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 2e-06, @@ -39746,36 +36731,6 @@ "supports_audio_input": false, "supports_response_schema": true }, - "moonshot/kimi-k2-0711-preview": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 6e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "moonshot/kimi-k2-0905-preview": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 6e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, "moonshot/kimi-k2.7-code": { "cache_read_input_token_cost": 1.9e-07, "input_cost_per_token": 9.5e-07, @@ -39794,21 +36749,6 @@ "supports_video_input": true, "supports_vision": true }, - "moonshot/kimi-k2-turbo-preview": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 1.15e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 8e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_web_search": true - }, "moonshot/kimi-k2.5": { "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 6e-07, @@ -39865,111 +36805,6 @@ "supports_video_input": true, "supports_vision": true }, - "moonshot/kimi-latest": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-01-28", - "input_cost_per_token": 2e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "moonshot/kimi-latest-128k": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-01-28", - "input_cost_per_token": 2e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "moonshot/kimi-latest-32k": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-01-28", - "input_cost_per_token": 1e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "moonshot/kimi-latest-8k": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-01-28", - "input_cost_per_token": 2e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "moonshot/kimi-thinking-preview": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2025-11-11", - "input_cost_per_token": 6e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_vision": true - }, - "moonshot/kimi-k2-thinking": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 6e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_web_search": true - }, - "moonshot/kimi-k2-thinking-turbo": { - "cache_read_input_token_cost": 1.5e-07, - "deprecation_date": "2026-05-25", - "input_cost_per_token": 1.15e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 8e-06, - "source": "https://platform.moonshot.ai/docs/pricing/chat#generation-model-kimi-k2", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_web_search": true - }, "moonshot/moonshot-v1-128k": { "input_cost_per_token": 2e-06, "litellm_provider": "moonshot", @@ -39983,19 +36818,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "moonshot/moonshot-v1-128k-0430": { - "deprecation_date": "2024-04-30", - "input_cost_per_token": 2e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 5e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true - }, "moonshot/moonshot-v1-128k-vision-preview": { "input_cost_per_token": 2e-06, "litellm_provider": "moonshot", @@ -40023,19 +36845,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "moonshot/moonshot-v1-32k-0430": { - "deprecation_date": "2024-04-30", - "input_cost_per_token": 1e-06, - "litellm_provider": "moonshot", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true - }, "moonshot/moonshot-v1-32k-vision-preview": { "input_cost_per_token": 1e-06, "litellm_provider": "moonshot", @@ -40063,19 +36872,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "moonshot/moonshot-v1-8k-0430": { - "deprecation_date": "2024-04-30", - "input_cost_per_token": 2e-07, - "litellm_provider": "moonshot", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://platform.moonshot.ai/docs/pricing", - "supports_function_calling": true, - "supports_tool_choice": true - }, "moonshot/moonshot-v1-8k-vision-preview": { "input_cost_per_token": 2e-07, "litellm_provider": "moonshot", @@ -41266,88 +38062,6 @@ "supports_vision": true, "supports_web_search": true }, - "o3-deep-research": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1e-05, - "input_cost_per_token_batches": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 4e-05, - "output_cost_per_token_batches": 2e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, - "o3-deep-research-2025-06-26": { - "cache_read_input_token_cost": 2.5e-06, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 1e-05, - "input_cost_per_token_batches": 5e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 4e-05, - "output_cost_per_token_batches": 2e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, "o3-mini": { "cache_read_input_token_cost": 5.5e-07, "deprecation_date": "2026-10-23", @@ -41533,88 +38247,6 @@ "supports_vision": true, "supports_web_search": true }, - "o4-mini-deep-research": { - "cache_read_input_token_cost": 5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 2e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 8e-06, - "output_cost_per_token_batches": 4e-06, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, - "o4-mini-deep-research-2025-06-26": { - "cache_read_input_token_cost": 5e-07, - "deprecation_date": "2026-07-23", - "input_cost_per_token": 2e-06, - "input_cost_per_token_batches": 1e-06, - "litellm_provider": "openai", - "max_input_tokens": 200000, - "max_output_tokens": 100000, - "max_tokens": 100000, - "mode": "responses", - "output_cost_per_token": 8e-06, - "output_cost_per_token_batches": 4e-06, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/batch", - "/v1/responses" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "text" - ], - "supports_function_calling": true, - "supports_native_streaming": true, - "supports_parallel_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true - }, "oci/meta.llama-3.1-8b-instruct": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", @@ -42587,32 +39219,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "openrouter/anthropic/claude-opus-4": { - "input_cost_per_image": 0.0048, - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_pdf_input": true, - "supports_response_schema": false, - "supports_web_search": true - }, "openrouter/anthropic/claude-opus-4.1": { "input_cost_per_image": 0.0048, "cache_creation_input_token_cost": 1.875e-05, @@ -42866,6 +39472,28 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "openrouter/anthropic/claude-opus-5.5": { + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "openrouter/bytedance/ui-tars-1.5-7b": { "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-07, @@ -43037,21 +39665,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.5526e-07, + "input_cost_per_token": 8.92272e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.91052e-06, + "output_cost_per_token": 1.784544e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.9605e-08, + "cache_read_input_token_cost": 7.4356e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -43079,44 +39707,27 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro-0813": { - "input_cost_per_token": 5.58624e-07, + "input_cost_per_token": 1.32e-06, "input_cost_per_token_cache_hit": 1.9272e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 1.675872e-06, + "output_cost_per_token": 3.96e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 1.86208e-08, - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.58624e-7,"output_cost_per_token":0.000001675872,"cache_read_input_token_cost":1.86208e-8}, + "cache_read_input_token_cost": 4.4e-08, + "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":6.6e-7,"output_cost_per_token":0.00000198,"cache_read_input_token_cost":2.2e-8}, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, "supports_web_search": false }, - "openrouter/google/gemini-2.0-flash-001": { - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7e-07, - "input_cost_per_token": 1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 4e-07, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true - }, "openrouter/google/gemini-2.5-flash": { "cache_creation_input_token_cost": 8.33333333333333e-08, "cache_read_input_audio_token_cost": 1e-07, @@ -43540,27 +40151,6 @@ "supports_vision": true, "supports_web_search": false }, - "openrouter/mistralai/mistral-large-2512": { - "cache_read_input_token_cost": 5.5e-08, - "input_cost_per_image": 0, - "input_cost_per_token": 5.5e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 209715, - "max_tokens": 209715, - "mode": "chat", - "output_cost_per_token": 1.65e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": false, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, "openrouter/mistralai/mistral-7b-instruct": { "input_cost_per_token": 1.3e-07, "litellm_provider": "openrouter", @@ -44178,19 +40768,19 @@ }, "openrouter/openai/gpt-oss-20b": { "cache_read_input_token_cost": 3e-08, - "input_cost_per_token": 3e-08, + "input_cost_per_token": 1.8e-08, "litellm_provider": "openrouter", "max_input_tokens": 131072, - "max_output_tokens": 117964, - "max_tokens": 117964, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 1.3e-07, + "output_cost_per_token": 9e-08, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": false, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -44812,26 +41402,6 @@ "max_tokens": 128000, "mode": "chat" }, - "openrouter/stealth/union-alpha": { - "deprecation_date": "2098-12-31", - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": false, - "supports_reasoning": false, - "supports_tool_choice": true, - "supports_response_schema": true, - "supports_vision": true, - "supports_web_search": false - }, "ovhcloud/DeepSeek-R1-Distill-Llama-70B": { "input_cost_per_token": 6.7e-07, "litellm_provider": "ovhcloud", @@ -46331,17 +42901,6 @@ "supports_reasoning": true, "supports_system_messages": true }, - "rerank-english-v2.0": { - "input_cost_per_query": 0.002, - "input_cost_per_token": 0.0, - "litellm_provider": "cohere", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "rerank", - "output_cost_per_token": 0.0, - "deprecation_date": "2025-04-30" - }, "rerank-english-v3.0": { "input_cost_per_query": 0.002, "input_cost_per_token": 0.0, @@ -46352,17 +42911,6 @@ "mode": "rerank", "output_cost_per_token": 0.0 }, - "rerank-multilingual-v2.0": { - "input_cost_per_query": 0.002, - "input_cost_per_token": 0.0, - "litellm_provider": "cohere", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "rerank", - "output_cost_per_token": 0.0, - "deprecation_date": "2025-04-30" - }, "rerank-multilingual-v3.0": { "input_cost_per_query": 0.002, "input_cost_per_token": 0.0, @@ -46503,31 +43051,6 @@ "output_cost_per_token": 7e-06, "source": "https://cloud.sambanova.ai/plans/pricing" }, - "sambanova/DeepSeek-R1-Distill-Llama-70B": { - "deprecation_date": "2026-03-20", - "input_cost_per_token": 7e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.4e-06, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, - "sambanova/DeepSeek-V3-0324": { - "deprecation_date": "2026-04-14", - "input_cost_per_token": 3e-06, - "litellm_provider": "sambanova", - "max_input_tokens": 32768, - "max_output_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 4.5e-06, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "sambanova/Llama-4-Maverick-17B-128E-Instruct": { "input_cost_per_token": 6.3e-07, "litellm_provider": "sambanova", @@ -46545,73 +43068,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "sambanova/Llama-4-Scout-17B-16E-Instruct": { - "deprecation_date": "2025-06-19", - "input_cost_per_token": 4e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "metadata": { - "notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount" - }, - "mode": "chat", - "output_cost_per_token": 7e-07, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "sambanova/Meta-Llama-3.1-405B-Instruct": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 5e-06, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-05, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "sambanova/Meta-Llama-3.1-8B-Instruct": { - "deprecation_date": "2026-04-14", - "input_cost_per_token": 1e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "sambanova/Meta-Llama-3.2-1B-Instruct": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 4e-08, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 8e-08, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, - "sambanova/Meta-Llama-3.2-3B-Instruct": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 8e-08, - "litellm_provider": "sambanova", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.6e-07, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, "sambanova/Meta-Llama-3.3-70B-Instruct": { "input_cost_per_token": 6e-07, "litellm_provider": "sambanova", @@ -46625,54 +43081,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "sambanova/Meta-Llama-Guard-3-8B": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 3e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, - "sambanova/QwQ-32B": { - "deprecation_date": "2025-06-25", - "input_cost_per_token": 5e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 16384, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 1e-06, - "source": "https://cloud.sambanova.ai/plans/pricing" - }, - "sambanova/Qwen2-Audio-7B-Instruct": { - "deprecation_date": "2025-06-19", - "input_cost_per_token": 5e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 4096, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 0.0001, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_audio_input": true - }, - "sambanova/Qwen3-32B": { - "deprecation_date": "2026-04-06", - "input_cost_per_token": 4e-07, - "litellm_provider": "sambanova", - "max_input_tokens": 8192, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://cloud.sambanova.ai/plans/pricing", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, "sambanova/DeepSeek-V3.1": { "max_tokens": 131072, "max_input_tokens": 131072, @@ -47265,27 +43673,6 @@ "mode": "image_generation", "output_cost_per_image": 0.14 }, - "standard/1024-x-1024/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 3.81469e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, - "standard/1024-x-1792/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 4.359e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, - "standard/1792-x-1024/dall-e-3": { - "deprecation_date": "2026-05-12", - "input_cost_per_pixel": 4.359e-08, - "litellm_provider": "openai", - "mode": "image_generation", - "output_cost_per_pixel": 0.0 - }, "linkup/search": { "input_cost_per_query": 0.00587, "litellm_provider": "linkup", @@ -47420,36 +43807,6 @@ "output_vector_size": 768, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, - "text-moderation-007": { - "deprecation_date": "2025-10-27", - "input_cost_per_token": 0.0, - "litellm_provider": "openai", - "max_input_tokens": 32768, - "max_output_tokens": 0, - "max_tokens": 0, - "mode": "moderation", - "output_cost_per_token": 0.0 - }, - "text-moderation-latest": { - "deprecation_date": "2025-10-27", - "input_cost_per_token": 0.0, - "litellm_provider": "openai", - "max_input_tokens": 32768, - "max_output_tokens": 0, - "max_tokens": 0, - "mode": "moderation", - "output_cost_per_token": 0.0 - }, - "text-moderation-stable": { - "deprecation_date": "2025-10-27", - "input_cost_per_token": 0.0, - "litellm_provider": "openai", - "max_input_tokens": 32768, - "max_output_tokens": 0, - "max_tokens": 0, - "mode": "moderation", - "output_cost_per_token": 0.0 - }, "text-multilingual-embedding-002": { "deprecation_date": "2027-04-01", "input_cost_per_character": 2.5e-08, @@ -47547,19 +43904,6 @@ "mode": "chat", "output_cost_per_token": 1e-07 }, - "together_ai/Qwen/Qwen2.5-72B-Instruct-Turbo": { - "deprecation_date": "2026-02-06", - "litellm_provider": "together_ai", - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "input_cost_per_token": 1.2e-06, - "output_cost_per_token": 1.2e-06, - "max_input_tokens": 131072, - "source": "https://api.together.ai/v1/models" - }, "together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo": { "litellm_provider": "together_ai", "mode": "chat", @@ -47572,87 +43916,6 @@ "max_input_tokens": 32768, "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { - "deprecation_date": "2026-07-10", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262000, - "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { - "deprecation_date": "2026-04-16", - "input_cost_per_token": 6.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": { - "deprecation_date": "2026-02-06", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 40000, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://www.together.ai/models/qwen3-235b-a22b-fp8-tput", - "supports_function_calling": false, - "supports_parallel_function_calling": false, - "supports_tool_choice": false - }, - "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": { - "deprecation_date": "2026-06-04", - "input_cost_per_token": 2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/deepseek-ai/DeepSeek-R1": { - "deprecation_date": "2026-05-14", - "input_cost_per_token": 3e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 128000, - "max_output_tokens": 20480, - "max_tokens": 20480, - "metadata": { - "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" - }, - "mode": "chat", - "output_cost_per_token": 7e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/deepseek-ai/DeepSeek-R1-0528-tput": { - "deprecation_date": "2026-02-03", - "input_cost_per_token": 5.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 2.19e-06, - "source": "https://www.together.ai/models/deepseek-r1-0528-throughput", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/deepseek-ai/DeepSeek-V3": { "input_cost_per_token": 1.25e-06, "litellm_provider": "together_ai", @@ -47669,33 +43932,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/deepseek-ai/DeepSeek-V3.1": { - "deprecation_date": "2026-05-14", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "max_tokens": 16384, - "metadata": { - "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" - }, - "mode": "chat", - "output_cost_per_token": 1.7e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "max_input_tokens": 131072, - "max_output_tokens": 16384 - }, - "together_ai/meta-llama/Llama-3.2-3B-Instruct-Turbo": { - "deprecation_date": "2026-03-06", - "litellm_provider": "together_ai", - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo": { "input_cost_per_token": 1.04e-06, "litellm_provider": "together_ai", @@ -47709,112 +43945,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo-Free": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 0, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 0, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { - "deprecation_date": "2026-03-31", - "input_cost_per_token": 2.7e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8.5e-07, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Llama-4-Scout-17B-16E-Instruct": { - "deprecation_date": "2026-02-06", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 5.9e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo": { - "deprecation_date": "2026-02-06", - "input_cost_per_token": 3.5e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 3.5e-06, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { - "deprecation_date": "2026-02-25", - "input_cost_per_token": 8.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8.8e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { - "deprecation_date": "2026-03-06", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/mistralai/Mistral-7B-Instruct-v0.1": { - "deprecation_date": "2025-11-13", - "litellm_provider": "together_ai", - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "input_cost_per_token": 2e-07, - "output_cost_per_token": 2e-07, - "max_input_tokens": 32768, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": { - "deprecation_date": "2026-04-02", - "litellm_provider": "together_ai", - "mode": "chat", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 3e-07, - "max_input_tokens": 32768, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1": { - "deprecation_date": "2026-04-16", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/moonshotai/Kimi-K2-Instruct": { "input_cost_per_token": 1e-06, "litellm_provider": "together_ai", @@ -47843,19 +43973,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/openai/gpt-oss-20b": { - "deprecation_date": "2026-09-14", - "input_cost_per_token": 5e-08, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/togethercomputer/CodeLlama-34b-Instruct": { "litellm_provider": "together_ai", "mode": "chat", @@ -47863,19 +43980,6 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true }, - "together_ai/zai-org/GLM-4.5-Air-FP8": { - "deprecation_date": "2026-04-02", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 1.1e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/zai-org/GLM-4.6": { "input_cost_per_token": 6e-07, "litellm_provider": "together_ai", @@ -47892,102 +43996,6 @@ "supports_reasoning": true, "supports_tool_choice": true }, - "together_ai/zai-org/GLM-4.7": { - "deprecation_date": "2026-04-02", - "input_cost_per_token": 4.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 202752, - "max_tokens": 202752, - "metadata": { - "successor": "together_ai/zai-org/GLM-5.2" - }, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_reasoning": true, - "supports_tool_choice": true - }, - "together_ai/moonshotai/Kimi-K2.5": { - "deprecation_date": "2026-05-21", - "input_cost_per_token": 5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 256000, - "max_tokens": 256000, - "metadata": { - "successor": "together_ai/moonshotai/Kimi-K3" - }, - "mode": "chat", - "output_cost_per_token": 2.8e-06, - "source": "https://www.together.ai/models/kimi-k2-5", - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_reasoning": true - }, - "together_ai/moonshotai/Kimi-K2-Instruct-0905": { - "deprecation_date": "2026-03-06", - "input_cost_per_token": 1e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "metadata": { - "successor": "together_ai/moonshotai/Kimi-K3" - }, - "mode": "chat", - "output_cost_per_token": 3e-06, - "source": "https://www.together.ai/models/kimi-k2-0905", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3-Next-80B-A3B-Instruct": { - "deprecation_date": "2026-04-02", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "metadata": { - "successor": "together_ai/Qwen/Qwen3.7-Plus" - }, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": { - "deprecation_date": "2026-02-25", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "metadata": { - "successor": "together_ai/Qwen/Qwen3.6-Plus" - }, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/Qwen/Qwen3.5-397B-A17B": { - "cache_read_input_token_cost": 3.5e-07, - "deprecation_date": "2026-06-29", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 3.6e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/MiniMaxAI/MiniMax-M3": { "cache_read_input_token_cost": 6e-08, "input_cost_per_token": 3e-07, @@ -48109,23 +44117,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/deepseek-ai/DeepSeek-V4-Pro": { - "deprecation_date": "2026-08-27", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.74e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 512000, - "max_tokens": 512000, - "mode": "chat", - "output_cost_per_token": 3.48e-06, - "source": "https://docs.together.ai/docs/serverless-models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": { "cache_read_input_token_cost": 1.3e-07, "deprecation_date": "2026-09-29", @@ -48142,52 +44133,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/google/gemma-3n-E4B-it": { - "deprecation_date": "2026-08-25", - "input_cost_per_token": 6e-08, - "litellm_provider": "together_ai", - "max_input_tokens": 32768, - "max_tokens": 32768, - "mode": "chat", - "output_cost_per_token": 1.2e-07, - "source": "https://docs.together.ai/docs/serverless-models" - }, - "together_ai/google/gemma-4-31B-it": { - "deprecation_date": "2026-09-14", - "input_cost_per_token": 3.9e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 9.7e-07, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "together_ai/intfloat/multilingual-e5-large-instruct": { - "deprecation_date": "2026-09-14", - "input_cost_per_token": 2e-08, - "litellm_provider": "together_ai", - "max_input_tokens": 514, - "max_tokens": 514, - "mode": "embedding", - "output_cost_per_token": 2e-08, - "output_vector_size": 1024, - "source": "https://docs.together.ai/docs/serverless-models" - }, - "together_ai/meta-llama/Llama-Guard-4-12B": { - "deprecation_date": "2026-08-25", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 1048576, - "max_tokens": 1048576, - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://docs.together.ai/docs/serverless-models" - }, "together_ai/meta-models/Muse-Glimmer-30B": { "cache_read_input_token_cost": 4e-08, "input_cost_per_token": 3.5e-07, @@ -48199,23 +44144,6 @@ "source": "https://api.together.ai/v1/models", "supports_prompt_caching": true }, - "together_ai/moonshotai/Kimi-K2.7-Code": { - "deprecation_date": "2026-08-27", - "cache_read_input_token_cost": 1.9e-07, - "input_cost_per_token": 9.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 4e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "together_ai/moonshotai/Kimi-K3": { "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, @@ -48238,33 +44166,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "together_ai/nvidia/nemotron-3-ultra-550b-a55b": { - "deprecation_date": "2026-08-27", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 512288, - "max_tokens": 512288, - "mode": "chat", - "output_cost_per_token": 3.6e-06, - "source": "https://api.together.ai/v1/models", - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true - }, - "together_ai/pearl-ai/gemma-4-31b-it": { - "deprecation_date": "2026-08-27", - "input_cost_per_token": 2.8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "max_tokens": 262144, - "mode": "chat", - "output_cost_per_token": 8.6e-07, - "source": "https://docs.together.ai/docs/serverless-models" - }, "together_ai/thinkingmachines/Inkling": { "cache_read_input_token_cost": 1.7e-07, "input_cost_per_token": 1e-06, @@ -48280,18 +44181,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "together_ai/thinkingmachines/Inkling-Small": { - "deprecation_date": "2026-09-14", - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 524288, - "max_tokens": 524288, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://api.together.ai/v1/models", - "supports_prompt_caching": true - }, "together_ai/zai-org/GLM-5.2": { "cache_read_input_token_cost": 2.6e-07, "input_cost_per_token": 1.4e-06, @@ -48438,22 +44327,6 @@ "supports_response_schema": true, "supports_tool_choice": true }, - "us.amazon.nova-premier-v1:0": { - "deprecation_date": "2026-09-14", - "input_cost_per_token": 2.5e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 1000000, - "max_output_tokens": 10000, - "max_tokens": 10000, - "mode": "chat", - "output_cost_per_token": 1.25e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_vision": true, - "cache_read_input_token_cost": 6.25e-07 - }, "us.amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 8e-07, @@ -48571,23 +44444,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "us.anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 2.5e-07, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.25e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 - }, "us.anthropic.claude-3-opus-20240229-v1:0": { "input_cost_per_token": 1.5e-05, "litellm_provider": "bedrock", @@ -48603,23 +44459,6 @@ "cache_read_input_token_cost": 1.5e-06, "cache_creation_input_token_cost": 1.875e-05 }, - "us.anthropic.claude-3-sonnet-20240229-v1:0": { - "deprecation_date": "2026-07-30", - "input_cost_per_token": 3e-06, - "litellm_provider": "bedrock", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-07, - "cache_creation_input_token_cost": 3.75e-06 - }, "us.anthropic.claude-opus-4-1-20250805-v1:0": { "cache_creation_input_token_cost": 1.875e-05, "cache_read_input_token_cost": 1.5e-06, @@ -48684,23 +44523,6 @@ "input_cost_per_token_batches": 1.65e-06, "output_cost_per_token_batches": 8.25e-06 }, - "us-gov.anthropic.claude-3-haiku-20240307-v1:0": { - "deprecation_date": "2026-09-10", - "input_cost_per_token": 3e-07, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "output_cost_per_token": 1.5e-06, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "cache_read_input_token_cost": 3e-08, - "cache_creation_input_token_cost": 3.75e-07 - }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, @@ -48832,6 +44654,41 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "us-gov.anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 6e-06, + "cache_creation_input_token_cost_above_1hr": 9.6e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 4.8e-06, + "litellm_provider": "bedrock_converse", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.4e-05, + "prompt_cache_min_tokens": 512, + "source": "https://aws.amazon.com/bedrock/pricing/", + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "thinking_always_on": true, + "supports_forced_tool_use": false + }, "us-gov.anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.5e-05, "cache_creation_input_token_cost_above_1hr": 2.4e-05, @@ -49828,34 +45685,6 @@ "output_cost_per_token": 9e-07, "supports_tool_choice": true }, - "vercel_ai_gateway/google/gemini-2.0-flash": { - "deprecation_date": "2026-06-01", - "input_cost_per_token": 1.5e-07, - "litellm_provider": "vercel_ai_gateway", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_vision": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true - }, - "vercel_ai_gateway/google/gemini-2.0-flash-lite": { - "deprecation_date": "2026-06-01", - "input_cost_per_token": 7.5e-08, - "litellm_provider": "vercel_ai_gateway", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "supports_vision": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_response_schema": true - }, "vercel_ai_gateway/google/gemini-2.5-flash": { "input_cost_per_token": 3e-07, "litellm_provider": "vercel_ai_gateway", @@ -50739,28 +46568,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "vertex_ai/claude-3-7-sonnet@20250219": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-05-11", - "input_cost_per_token": 3e-06, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true - }, "vertex_ai/claude-3-haiku": { "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-anthropic_models", @@ -50839,72 +46646,6 @@ "supports_tool_choice": true, "supports_vision": true }, - "vertex_ai/claude-opus-4": { - "deprecation_date": "2026-05-14", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "vertex_ai/claude-opus-4-1": { - "deprecation_date": "2026-08-05", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "input_cost_per_token_batches": 7.5e-06, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "output_cost_per_token_batches": 3.75e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, - "vertex_ai/claude-opus-4-1@20250805": { - "deprecation_date": "2026-08-05", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "input_cost_per_token_batches": 7.5e-06, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "output_cost_per_token_batches": 3.75e-05, - "supports_assistant_prefill": true, - "supports_function_calling": true, - "supports_tool_choice": true, - "supports_vision": true - }, "vertex_ai/claude-opus-4-5": { "deprecation_date": "2026-11-24", "cache_creation_input_token_cost": 6.25e-06, @@ -51310,6 +47051,78 @@ "supports_max_reasoning_effort": true, "prompt_cache_min_tokens": 512 }, + "vertex_ai/claude-opus-5-5": { + "regional_endpoint_uplift_multiplier": 1.1, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 + }, + "vertex_ai/claude-opus-5-5@default": { + "regional_endpoint_uplift_multiplier": 1.1, + "supports_mid_conversation_system": true, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 512 + }, "vertex_ai/claude-opus-4-8": { "deprecation_date": "2027-05-28", "regional_endpoint_uplift_multiplier": 1.1, @@ -51505,98 +47318,6 @@ "supports_native_streaming": true, "prompt_cache_min_tokens": 1024 }, - "vertex_ai/claude-opus-4@20250514": { - "deprecation_date": "2026-05-14", - "cache_creation_input_token_cost": 1.875e-05, - "cache_creation_input_token_cost_above_1hr": 3e-05, - "cache_read_input_token_cost": 1.5e-06, - "input_cost_per_token": 1.5e-05, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "output_cost_per_token": 7.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "vertex_ai/claude-sonnet-4": { - "deprecation_date": "2026-05-14", - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 1000000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, - "vertex_ai/claude-sonnet-4@20250514": { - "deprecation_date": "2026-05-14", - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "litellm_provider": "vertex_ai-anthropic_models", - "max_input_tokens": 1000000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.01, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.01 - }, - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "prompt_cache_min_tokens": 1024 - }, "vertex_ai/mistralai/codestral-2@001": { "input_cost_per_token": 3e-07, "litellm_provider": "vertex_ai-mistral_models", @@ -51785,6 +47506,7 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -51809,7 +47531,8 @@ "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05, "supports_reasoning": false, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "cache_read_input_token_cost_batches": 1e-07 }, "vertex_ai/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -51829,6 +47552,7 @@ }, "vertex_ai/gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -51846,7 +47570,8 @@ "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, "supports_reasoning": false, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "cache_read_input_token_cost_batches": 2.5e-08 }, "vertex_ai/gemini-3.1-flash-image-preview": { "input_cost_per_image": 0.00056, @@ -51864,6 +47589,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -51901,7 +47627,8 @@ "supports_response_schema": false, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost_batches": 1.25e-08 }, "vertex_ai/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, @@ -51956,6 +47683,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -52011,11 +47739,13 @@ }, "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, - "input_cost_per_audio_token_batches": 2.5e-07 + "input_cost_per_audio_token_batches": 2.5e-07, + "cache_read_input_token_cost_batches": 1.25e-08 }, "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -52069,10 +47799,12 @@ "search_context_size_high": 0.014 }, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "cache_read_input_token_cost_batches": 1.5e-08 }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -52085,63 +47817,8 @@ "output_cost_per_image_token": 0.00012, "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 6e-06, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" - }, - "vertex_ai/imagegeneration@006": { - "deprecation_date": "2025-09-24", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-3.0-fast-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-3.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-3.0-generate-002": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-3.0-capability-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/image/edit-insert-objects" - }, - "vertex_ai/imagen-4.0-fast-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.02, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-4.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.04, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "vertex_ai/imagen-4.0-ultra-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-image-models", - "mode": "image_generation", - "output_cost_per_image": 0.06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "cache_read_input_token_cost_batches": 1e-07 }, "vertex_ai/jamba-1.5": { "input_cost_per_token": 2e-07, @@ -52887,51 +48564,6 @@ "supports_function_calling": true, "supports_tool_choice": true }, - "vertex_ai/veo-2.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-video-models", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "video_generation", - "output_cost_per_second": 0.35, - "source": "https://ai.google.dev/gemini-api/docs/video", - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "video" - ] - }, - "vertex_ai/veo-3.0-fast-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-video-models", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "video_generation", - "output_cost_per_second": 0.15, - "source": "https://ai.google.dev/gemini-api/docs/video", - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "video" - ] - }, - "vertex_ai/veo-3.0-generate-001": { - "deprecation_date": "2026-06-30", - "litellm_provider": "vertex_ai-video-models", - "max_input_tokens": 1024, - "max_tokens": 1024, - "mode": "video_generation", - "output_cost_per_second": 0.4, - "source": "https://ai.google.dev/gemini-api/docs/video", - "supported_modalities": [ - "text" - ], - "supported_output_modalities": [ - "video" - ] - }, "vertex_ai/veo-3.1-generate-preview": { "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, @@ -53220,59 +48852,6 @@ "mode": "chat", "source": "https://wandb.ai/site/pricing/tokens/" }, - "wandb/zai-org/GLM-4.5": { - "deprecation_date": "2026-03-04", - "supports_reasoning": true, - "max_tokens": 131072, - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "input_cost_per_token": 0.055, - "output_cost_per_token": 0.2, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/Qwen/Qwen3-235B-A22B-Instruct-2507": { - "deprecation_date": "2026-08-04", - "max_tokens": 262144, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 1e-07, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/Qwen/Qwen3-Coder-480B-A35B-Instruct": { - "deprecation_date": "2026-08-25", - "max_tokens": 262144, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 1e-06, - "output_cost_per_token": 1.5e-06, - "litellm_provider": "wandb", - "mode": "chat", - "source": "https://wandb.ai/site/pricing/tokens/" - }, - "wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": { - "deprecation_date": "2026-08-04", - "supports_reasoning": true, - "max_tokens": 262144, - "max_input_tokens": 262144, - "max_output_tokens": 262144, - "input_cost_per_token": 1e-07, - "output_cost_per_token": 1e-07, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/moonshotai/Kimi-K2-Instruct": { - "deprecation_date": "2026-03-04", - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 6e-07, - "output_cost_per_token": 2.5e-06, - "litellm_provider": "wandb", - "mode": "chat" - }, "wandb/moonshotai/Kimi-K2.5": { "max_tokens": 262144, "max_input_tokens": 262144, @@ -53288,20 +48867,6 @@ "supports_response_schema": true, "supports_vision": true }, - "wandb/MiniMaxAI/MiniMax-M2.5": { - "deprecation_date": "2026-08-25", - "max_tokens": 197000, - "max_input_tokens": 197000, - "max_output_tokens": 197000, - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "wandb", - "mode": "chat", - "source": "https://wandb.ai/inference/coreweave/cw_MiniMaxAI_MiniMax-M2.5", - "supports_function_calling": true, - "supports_reasoning": true, - "supports_response_schema": true - }, "wandb/meta-llama/Llama-3.1-8B-Instruct": { "max_tokens": 128000, "max_input_tokens": 131000, @@ -53323,27 +48888,6 @@ "mode": "chat", "source": "https://wandb.ai/site/pricing/tokens/" }, - "wandb/deepseek-ai/DeepSeek-R1-0528": { - "deprecation_date": "2026-03-04", - "supports_reasoning": true, - "max_tokens": 161000, - "max_input_tokens": 161000, - "max_output_tokens": 161000, - "input_cost_per_token": 1.35e-06, - "output_cost_per_token": 5.4e-06, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/deepseek-ai/DeepSeek-V3-0324": { - "deprecation_date": "2026-03-04", - "max_tokens": 161000, - "max_input_tokens": 161000, - "max_output_tokens": 161000, - "input_cost_per_token": 1.14e-06, - "output_cost_per_token": 2.75e-06, - "litellm_provider": "wandb", - "mode": "chat" - }, "wandb/meta-llama/Llama-3.3-70B-Instruct": { "max_tokens": 128000, "max_input_tokens": 128000, @@ -53354,26 +48898,6 @@ "mode": "chat", "source": "https://wandb.ai/site/pricing/tokens/" }, - "wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": { - "deprecation_date": "2026-04-21", - "max_tokens": 64000, - "max_input_tokens": 64000, - "max_output_tokens": 64000, - "input_cost_per_token": 1.7e-07, - "output_cost_per_token": 6.6e-07, - "litellm_provider": "wandb", - "mode": "chat" - }, - "wandb/microsoft/Phi-4-mini-instruct": { - "deprecation_date": "2026-08-04", - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 0.008, - "output_cost_per_token": 0.035, - "litellm_provider": "wandb", - "mode": "chat" - }, "watsonx/ibm/granite-3-8b-instruct": { "input_cost_per_token": 2e-07, "litellm_provider": "watsonx", @@ -53774,440 +49298,6 @@ "deprecation_date": "2027-02-26", "source": "https://developers.openai.com/api/docs/pricing" }, - "xai/grok-3": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-beta": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-fast-beta": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-fast-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini": { - "cache_read_input_token_cost": 2e-07, - "deprecation_date": "2026-02-28", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-beta": { - "cache_read_input_token_cost": 2e-07, - "deprecation_date": "2026-02-28", - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-fast": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-02-28", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-fast-beta": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-02-28", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-fast-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-02-28", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-3-mini-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 131072, - "max_output_tokens": 131072, - "max_tokens": 131072, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://x.ai/api#pricing", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-02-28", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4": { - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-fast-reasoning": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-fast-non-reasoning": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-0709": { - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-latest": { - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_tool_choice": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast-reasoning": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast-reasoning-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast-non-reasoning": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, - "xai/grok-4-1-fast-non-reasoning-latest": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1.25e-06, - "litellm_provider": "xai", - "max_input_tokens": 2000000.0, - "max_output_tokens": 2000000.0, - "max_tokens": 2000000.0, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "deprecation_date": "2026-05-15", - "input_cost_per_token_above_200k_tokens": 2.5e-06, - "output_cost_per_token_above_200k_tokens": 5e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07 - }, "xai/grok-4.20-multi-agent-beta-0309": { "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 1.25e-06, @@ -54217,7 +49307,7 @@ "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54227,6 +49317,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true, "supported_endpoints": [ "/v1/responses" @@ -54241,7 +49332,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54251,6 +49342,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-0309-reasoning": { @@ -54262,7 +49354,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -54271,6 +49363,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, @@ -54283,7 +49376,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, @@ -54292,11 +49385,13 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.3": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "xai", @@ -54306,7 +49401,7 @@ "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54318,6 +49413,7 @@ "xai/grok-4.3-latest": { "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "xai", @@ -54327,7 +49423,7 @@ "mode": "chat", "output_cost_per_token": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54339,6 +49435,7 @@ "xai/grok-4.5": { "cache_read_input_token_cost": 3e-07, "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54348,7 +49445,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54360,6 +49457,7 @@ "xai/grok-4.5-latest": { "cache_read_input_token_cost": 3e-07, "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54369,7 +49467,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54381,6 +49479,7 @@ "xai/grok-build-latest": { "cache_read_input_token_cost": 3e-07, "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54390,7 +49489,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/developers/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54402,6 +49501,7 @@ "xai/grok-4.6": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54411,7 +49511,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/developers/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54423,6 +49523,7 @@ "xai/grok-4.7": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_image_token": 2e-06, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, "litellm_provider": "xai", @@ -54432,7 +49533,7 @@ "mode": "chat", "output_cost_per_token": 6e-06, "output_cost_per_token_above_200k_tokens": 1.2e-05, - "source": "https://docs.x.ai/developers/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -54441,69 +49542,6 @@ "supports_vision": true, "supports_web_search": true }, - "xai/grok-code-fast": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "input_cost_per_token_above_200k_tokens": 2e-06, - "output_cost_per_token_above_200k_tokens": 4e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "supports_response_schema": true, - "supports_vision": true, - "deprecation_date": "2026-05-15" - }, - "xai/grok-code-fast-1": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "input_cost_per_token_above_200k_tokens": 2e-06, - "output_cost_per_token_above_200k_tokens": 4e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "supports_response_schema": true, - "supports_vision": true, - "deprecation_date": "2026-05-15" - }, - "xai/grok-code-fast-1-0825": { - "cache_read_input_token_cost": 2e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "xai", - "max_input_tokens": 256000, - "max_output_tokens": 256000, - "max_tokens": 256000, - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://docs.x.ai/docs/models", - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_tool_choice": true, - "input_cost_per_token_above_200k_tokens": 2e-06, - "output_cost_per_token_above_200k_tokens": 4e-06, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "supports_response_schema": true, - "supports_vision": true, - "deprecation_date": "2026-05-15" - }, "zai.glm-4.7": { "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", @@ -57246,30 +52284,6 @@ "supports_reasoning": true, "supports_vision": true }, - "scaleway/google/gemma-3-27b-it": { - "input_cost_per_token": 2.5e-07, - "litellm_provider": "scaleway", - "max_input_tokens": 40000, - "max_output_tokens": 8192, - "max_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 5e-07, - "supports_function_calling": true, - "supports_vision": true, - "deprecation_date": "2026-08-01" - }, - "scaleway/hcompany/holo2-30b-a3b": { - "input_cost_per_token": 3e-07, - "litellm_provider": "scaleway", - "max_input_tokens": 22000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 7e-07, - "supports_reasoning": true, - "supports_vision": true, - "deprecation_date": "2026-08-09" - }, "scaleway/mistralai/mistral-medium-3.5-128b": { "input_cost_per_token": 1.5e-06, "litellm_provider": "scaleway", @@ -57283,29 +52297,6 @@ "supports_vision": true, "supports_tool_choice": true }, - "scaleway/mistralai/devstral-2-123b-instruct-2512": { - "input_cost_per_token": 4e-07, - "litellm_provider": "scaleway", - "max_input_tokens": 200000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 2e-06, - "supports_function_calling": true, - "deprecation_date": "2026-08-01" - }, - "scaleway/mistralai/voxtral-small-24b-2507": { - "input_cost_per_audio_token": 1.5e-07, - "input_cost_per_token": 1.5e-07, - "litellm_provider": "scaleway", - "max_input_tokens": 32000, - "max_output_tokens": 16384, - "max_tokens": 16384, - "mode": "chat", - "output_cost_per_token": 3.5e-07, - "supports_audio_input": true, - "deprecation_date": "2026-08-01" - }, "scaleway/mistralai/mistral-small-3.2-24b-instruct-2506": { "input_cost_per_token": 1.5e-07, "litellm_provider": "scaleway", @@ -58865,26 +53856,6 @@ "/v1/audio/speech" ] }, - "gpt-4o-mini-tts-2025-03-20": { - "deprecation_date": "2026-07-23", - "input_cost_per_token": 6e-07, - "litellm_provider": "openai", - "mode": "audio_speech", - "output_cost_per_audio_token": 1.2e-05, - "output_cost_per_second": 0.00025, - "output_cost_per_token": 1e-05, - "source": "https://developers.openai.com/api/docs/pricing", - "supported_endpoints": [ - "/v1/audio/speech" - ], - "supported_modalities": [ - "text", - "audio" - ], - "supported_output_modalities": [ - "audio" - ] - }, "gpt-4o-mini-tts-2025-12-15": { "input_cost_per_token": 6e-07, "litellm_provider": "openai", @@ -58988,41 +53959,6 @@ "supports_none_reasoning_effort": false, "supports_xhigh_reasoning_effort": false }, - "gpt-realtime-mini-2025-10-06": { - "cache_creation_input_audio_token_cost": 3e-07, - "cache_read_input_audio_token_cost": 3e-07, - "cache_read_input_token_cost": 6e-08, - "deprecation_date": "2026-07-23", - "input_cost_per_audio_token": 1e-05, - "input_cost_per_image_token": 8e-07, - "input_cost_per_token": 6e-07, - "litellm_provider": "openai", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 2.4e-06, - "source": "https://developers.openai.com/api/docs/pricing", - "supported_endpoints": [ - "/v1/realtime" - ], - "supported_modalities": [ - "text", - "image", - "audio" - ], - "supported_output_modalities": [ - "text", - "audio" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_system_messages": true, - "supports_tool_choice": true - }, "gpt-realtime-mini-2025-12-15": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, @@ -59118,6 +54054,7 @@ }, "chatgpt-image-latest": { "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", "input_cost_per_image_token": 8e-06, "input_cost_per_image_token_batches": 4e-06, @@ -59176,42 +54113,6 @@ "tpm": 250000, "rpm": 10 }, - "gemini/gemini-2.0-flash-lite-001": { - "cache_read_input_token_cost": 1.875e-08, - "deprecation_date": "2026-06-01", - "input_cost_per_audio_token": 7.5e-08, - "input_cost_per_token": 7.5e-08, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 8192, - "mode": "chat", - "output_cost_per_token": 3e-07, - "rpm": 4000, - "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.0-flash-lite", - "supported_modalities": [ - "text", - "image", - "audio", - "video" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_output": true, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_response_schema": true, - "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": true, - "tpm": 4000000, - "search_context_cost_per_query": { - "search_context_size_low": 0.035, - "search_context_size_medium": 0.035, - "search_context_size_high": 0.035 - } - }, "gemini-2.5-flash-native-audio-latest": { "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, @@ -61978,7 +56879,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, @@ -61987,6 +56888,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent-0309": { @@ -61998,7 +56900,7 @@ "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": false, "supports_prompt_caching": true, "supports_reasoning": true, @@ -62008,6 +56910,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true, "supported_endpoints": [ "/v1/responses" @@ -62022,7 +56925,7 @@ "max_tokens": 256000, "mode": "chat", "output_cost_per_token": 2e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -62030,6 +56933,7 @@ "input_cost_per_token_above_200k_tokens": 2e-06, "output_cost_per_token_above_200k_tokens": 4e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, "supports_response_schema": true, "supports_vision": true }, @@ -62105,6 +57009,7 @@ "supports_audio_output": true }, "claude-mythos-5": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -62144,6 +57049,7 @@ } }, "claude-mythos-5-1": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 2.5e-07, @@ -62184,6 +57090,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-mythos-preview": { + "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -65225,7 +60132,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -65234,6 +60141,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, @@ -65246,7 +60154,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -65255,6 +60163,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, @@ -65267,7 +60176,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -65276,6 +60185,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_prompt_caching": true, "supports_response_schema": true }, @@ -65362,23 +60272,6 @@ "image" ] }, - "xai/grok-imagine-image-pro": { - "input_cost_per_image": 0.05, - "litellm_provider": "xai", - "mode": "image_generation", - "source": "https://docs.x.ai/docs/models", - "supported_endpoints": [ - "/v1/images/generations" - ], - "supported_modalities": [ - "text", - "image" - ], - "supported_output_modalities": [ - "image" - ], - "deprecation_date": "2026-05-15" - }, "xai/grok-imagine-image-2.0": { "input_cost_per_image": 0.06, "litellm_provider": "xai", @@ -65519,7 +60412,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, @@ -65528,6 +60421,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-non-reasoning-latest": { @@ -65539,7 +60433,7 @@ "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, @@ -65548,6 +60442,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent": { @@ -65559,7 +60454,7 @@ "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supported_endpoints": [ "/v1/responses" ], @@ -65572,6 +60467,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "xai/grok-4.20-multi-agent-latest": { @@ -65583,7 +60479,7 @@ "max_tokens": 1000000, "mode": "responses", "output_cost_per_token": 2.5e-06, - "source": "https://docs.x.ai/docs/models", + "source": "https://api.x.ai/v1/language-models", "supported_endpoints": [ "/v1/responses" ], @@ -65596,6 +60492,7 @@ "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, "supports_response_schema": true }, "groq/qwen/qwen3.8-27b": { @@ -65992,16 +60889,6 @@ "output_cost_per_token": 2.82e-07, "source": "https://api.together.ai/v1/models" }, - "together_ai/moonshotai/Kimi-K2.6": { - "deprecation_date": "2026-08-19", - "input_cost_per_token": 1.2e-06, - "output_cost_per_token": 4.5e-06, - "cache_read_input_token_cost": 2e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, "together_ai/moonshotai/Kimi-K2.5-fp4": { "input_cost_per_token": 5e-07, "output_cost_per_token": 2.8e-06, @@ -66019,25 +60906,6 @@ "mode": "chat", "source": "https://api.together.ai/v1/models" }, - "together_ai/zai-org/GLM-5": { - "deprecation_date": "2026-06-22", - "input_cost_per_token": 1e-06, - "output_cost_per_token": 3.2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 202752, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, - "together_ai/zai-org/GLM-5.1": { - "deprecation_date": "2026-07-10", - "input_cost_per_token": 1.4e-06, - "output_cost_per_token": 4.4e-06, - "cache_read_input_token_cost": 2.6e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 202752, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, "together_ai/deepseek-ai/DeepSeek-R1-0528": { "input_cost_per_token": 3e-06, "output_cost_per_token": 7e-06, @@ -66046,33 +60914,6 @@ "mode": "chat", "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen3-Coder-Next-FP8": { - "deprecation_date": "2026-05-14", - "input_cost_per_token": 5e-07, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen3-VL-32B-Instruct": { - "deprecation_date": "2026-02-25", - "input_cost_per_token": 5e-07, - "output_cost_per_token": 1.5e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen3-VL-8B-Instruct": { - "deprecation_date": "2026-04-16", - "input_cost_per_token": 1.8e-07, - "output_cost_per_token": 6.8e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 262144, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, "together_ai/mistralai/Ministral-3-14B-Instruct-2512": { "input_cost_per_token": 2e-07, "output_cost_per_token": 2e-07, @@ -66097,15 +60938,6 @@ "mode": "chat", "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/QwQ-32B": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 1.2e-06, - "output_cost_per_token": 1.2e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 131072, - "mode": "chat", - "source": "https://api.together.ai/v1/models" - }, "cerebras/gemma-4-31b": { "input_cost_per_token": 9.9e-07, "litellm_provider": "cerebras", @@ -66357,6 +61189,41 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "bedrock/us-gov-west-1/anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 6e-06, + "cache_creation_input_token_cost_above_1hr": 9.6e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 4.8e-06, + "litellm_provider": "bedrock", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.4e-05, + "prompt_cache_min_tokens": 512, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-gov-west-1/anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.5e-05, "cache_creation_input_token_cost_above_1hr": 2.4e-05, @@ -66563,6 +61430,41 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "bedrock/us-gov-east-1/anthropic.claude-opus-5-5": { + "bedrock_converse_supports_strict_tools": false, + "cache_creation_input_token_cost": 6e-06, + "cache_creation_input_token_cost_above_1hr": 9.6e-06, + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 4.8e-06, + "litellm_provider": "bedrock", + "supports_tool_search": true, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.4e-05, + "prompt_cache_min_tokens": 512, + "supports_adaptive_thinking": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_mid_conversation_system": true, + "supports_native_structured_output": false, + "supports_output_config": true, + "supports_parallel_tool_use_config": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "thinking_always_on": true, + "supports_forced_tool_use": false, + "source": "https://aws.amazon.com/bedrock/pricing/" + }, "bedrock/us-gov-east-1/anthropic.claude-fable-5-1": { "cache_creation_input_token_cost": 1.5e-05, "cache_creation_input_token_cost_above_1hr": 2.4e-05, @@ -68085,7 +62987,7 @@ "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://www.baseten.co/pricing/", + "source": "https://inference.baseten.co/v1/models", "supported_modalities": [ "text", "image" @@ -68095,6 +62997,7 @@ ], "supports_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true @@ -68252,9 +63155,9 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3": { - "input_cost_per_token": 9.1e-07, - "output_cost_per_token": 2.86e-06, - "cache_read_input_token_cost": 1.69e-07, + "input_cost_per_token": 6.538e-07, + "output_cost_per_token": 2.0548e-06, + "cache_read_input_token_cost": 1.2142e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 131072, @@ -68620,7 +63523,7 @@ }, "openrouter/moonshotai/kimi-k2.7-code": { "input_cost_per_token": 7.062e-07, - "output_cost_per_token": 3.21e-06, + "output_cost_per_token": 3.3e-06, "cache_read_input_token_cost": 1.8e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, @@ -68880,13 +63783,13 @@ "supports_web_search": false }, "openrouter/qwen/qwen3.6-27b": { - "input_cost_per_token": 3e-07, - "output_cost_per_token": 2e-06, - "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 3.2e-07, + "output_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 1.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_output_tokens": 262140, + "max_tokens": 262140, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -68941,9 +63844,9 @@ "supports_web_search": true }, "openrouter/deepseek/deepseek-v4-flash": { - "input_cost_per_token": 8.8606e-08, - "output_cost_per_token": 1.77212e-07, - "cache_read_input_token_cost": 1.77212e-08, + "input_cost_per_token": 4.9e-08, + "output_cost_per_token": 9.8e-08, + "cache_read_input_token_cost": 9.8e-09, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, @@ -69803,8 +64706,8 @@ "output_cost_per_token": 1.2e-06, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 235929, + "max_tokens": 235929, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -70847,38 +65750,6 @@ "output_cost_per_token": 1.5e-07, "source": "https://api.together.ai/v1/models" }, - "together_ai/deepseek-ai/deepseek-coder-33b-instruct": { - "deprecation_date": "2024-08-22", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Llama-70B": { - "deprecation_date": "2025-12-23", - "input_cost_per_token": 2e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 2e-06, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 1.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1.8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 1.6e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1.6e-06, - "source": "https://api.together.ai/v1/models" - }, "vertex_ai/gemini-2.5-flash-native-audio": { "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, @@ -70897,16 +65768,6 @@ "output_cost_per_token": 1e-05, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, - "vertex_ai/gemini-3.1-flash-live-preview": { - "input_cost_per_audio_token": 3e-06, - "input_cost_per_second": 8.33333333333e-05, - "input_cost_per_token": 7.5e-07, - "litellm_provider": "vertex_ai", - "mode": "realtime", - "output_cost_per_audio_token": 1.2e-05, - "output_cost_per_token": 4.5e-06, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" - }, "vertex_ai/gemini-3.1-flash-tts-preview": { "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, @@ -70939,16 +65800,6 @@ "output_cost_per_token": 9e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, - "vertex_ai/gemini-robotics-er-2": { - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, - "litellm_provider": "vertex_ai", - "mode": "chat", - "output_cost_per_token": 5e-06, - "output_cost_per_token_batches": 2.5e-06, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" - }, "vertex_ai/gemma-4-26b-a4b-it": { "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, @@ -70957,14 +65808,6 @@ "output_cost_per_token": 6e-07, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, - "together_ai/google/gemma-2-27b-it": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, "gpt-5.5-cyber": { "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 1.25e-05, @@ -70982,14 +65825,6 @@ "output_cost_per_token": 2.5e-05, "source": "https://developers.openai.com/api/docs/pricing" }, - "together_ai/meta-llama/Llama-3-8b-chat-hf": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/meta-llama/Llama-3.1-405B-Instruct": { "input_cost_per_token": 3.5e-06, "litellm_provider": "together_ai", @@ -71011,38 +65846,6 @@ "output_cost_per_token": 6e-08, "source": "https://api.together.ai/v1/models" }, - "together_ai/meta-llama/Meta-Llama-3-70B-Instruct-Turbo": { - "deprecation_date": "2025-12-23", - "input_cost_per_token": 8.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8.8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/meta-llama/Meta-Llama-3-8B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 2e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 2e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 6e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 8.8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8.8e-07, - "source": "https://api.together.ai/v1/models" - }, "together_ai/Qwen/Qwen2-1.5B-Instruct": { "input_cost_per_token": 2e-08, "litellm_provider": "together_ai", @@ -71050,22 +65853,6 @@ "output_cost_per_token": 2e-08, "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen2-72B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 9e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 9e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen2-VL-72B-Instruct": { - "deprecation_date": "2025-08-28", - "input_cost_per_token": 1.2e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://api.together.ai/v1/models" - }, "together_ai/Qwen/Qwen2.5-14B-Instruct": { "input_cost_per_token": 8e-07, "litellm_provider": "together_ai", @@ -71080,22 +65867,6 @@ "output_cost_per_token": 1.2e-06, "source": "https://api.together.ai/v1/models" }, - "together_ai/Qwen/Qwen2.5-Coder-32B-Instruct": { - "deprecation_date": "2025-11-13", - "input_cost_per_token": 8e-07, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8e-07, - "source": "https://api.together.ai/v1/models" - }, - "together_ai/Qwen/Qwen2.5-VL-72B-Instruct": { - "deprecation_date": "2026-01-05", - "input_cost_per_token": 1.95e-06, - "litellm_provider": "together_ai", - "mode": "chat", - "output_cost_per_token": 8e-06, - "source": "https://api.together.ai/v1/models" - }, "azure/eu/codex-mini": { "deprecation_date": "2026-11-15", "cache_read_input_token_cost": 4.13e-07, @@ -71164,6 +65935,7 @@ "azure/eu/gpt-5": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_batches": 6.875e-08, "cache_read_input_token_cost_priority": 2.75e-07, "input_cost_per_token": 1.375e-06, "input_cost_per_token_batches": 6.875e-07, @@ -71187,6 +65959,7 @@ "azure/eu/gpt-5-mini": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 2.75e-08, + "cache_read_input_token_cost_batches": 1.375e-08, "cache_read_input_token_cost_priority": 4.95e-08, "input_cost_per_token": 2.75e-07, "input_cost_per_token_batches": 1.375e-07, @@ -71201,6 +65974,7 @@ "azure/eu/gpt-5-nano": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 5.5e-09, + "cache_read_input_token_cost_batches": 2.75e-09, "input_cost_per_token": 5.5e-08, "input_cost_per_token_batches": 2.75e-08, "litellm_provider": "azure", @@ -71231,6 +66005,7 @@ "azure/eu/gpt-5.2": { "deprecation_date": "2027-06-08", "cache_read_input_token_cost": 1.925e-07, + "cache_read_input_token_cost_batches": 9.625e-08, "cache_read_input_token_cost_priority": 3.85e-07, "input_cost_per_token": 1.925e-06, "input_cost_per_token_batches": 9.625e-07, @@ -71242,15 +66017,6 @@ "output_cost_per_token_priority": 3.08e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/eu/gpt-5.2-chat": { - "deprecation_date": "2026-06-29", - "cache_read_input_token_cost": 1.925e-07, - "input_cost_per_token": 1.925e-06, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 1.54e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/eu/gpt-5.2-codex": { "deprecation_date": "2027-07-13", "cache_read_input_token_cost": 1.925e-07, @@ -71269,15 +66035,6 @@ "output_cost_per_token_batches": 9.24e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/eu/gpt-5.3-chat": { - "deprecation_date": "2026-06-29", - "cache_read_input_token_cost": 1.925e-07, - "input_cost_per_token": 1.925e-06, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 1.54e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/eu/gpt-5.3-codex": { "deprecation_date": "2027-08-24", "cache_read_input_token_cost": 1.925e-07, @@ -71293,6 +66050,7 @@ "azure/eu/gpt-5.4-mini": { "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 8.25e-08, + "cache_read_input_token_cost_batches": 4.125e-08, "cache_read_input_token_cost_priority": 1.65e-07, "input_cost_per_token": 8.25e-07, "input_cost_per_token_batches": 4.125e-07, @@ -71307,6 +66065,7 @@ "azure/eu/gpt-5.4-nano": { "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2.2e-08, + "cache_read_input_token_cost_batches": 1.1e-08, "input_cost_per_token": 2.2e-07, "input_cost_per_token_batches": 1.1e-07, "litellm_provider": "azure", @@ -71319,11 +66078,13 @@ "deprecation_date": "2027-09-07", "input_cost_per_token": 3.3e-05, "input_cost_per_token_above_272k_tokens": 6.6e-05, + "input_cost_per_token_above_272k_tokens_batches": 3.3e-05, "input_cost_per_token_batches": 1.65e-05, "litellm_provider": "azure", "mode": "chat", "output_cost_per_token": 0.000198, "output_cost_per_token_above_272k_tokens": 0.000297, + "output_cost_per_token_above_272k_tokens_batches": 0.0001485, "output_cost_per_token_batches": 9.9e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, @@ -71350,14 +66111,6 @@ "output_cost_per_token_batches": 2.42e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/eu/o1-preview": { - "cache_read_input_token_cost": 8.25e-06, - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/eu/o3-2025-04-16": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, @@ -71518,6 +66271,7 @@ "azure/us/gpt-5": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 1.375e-07, + "cache_read_input_token_cost_batches": 6.875e-08, "cache_read_input_token_cost_priority": 2.75e-07, "input_cost_per_token": 1.375e-06, "input_cost_per_token_batches": 6.875e-07, @@ -71541,6 +66295,7 @@ "azure/us/gpt-5-mini": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 2.75e-08, + "cache_read_input_token_cost_batches": 1.375e-08, "cache_read_input_token_cost_priority": 4.95e-08, "input_cost_per_token": 2.75e-07, "input_cost_per_token_batches": 1.375e-07, @@ -71555,6 +66310,7 @@ "azure/us/gpt-5-nano": { "deprecation_date": "2027-02-09", "cache_read_input_token_cost": 5.5e-09, + "cache_read_input_token_cost_batches": 2.75e-09, "input_cost_per_token": 5.5e-08, "input_cost_per_token_batches": 2.75e-08, "litellm_provider": "azure", @@ -71585,6 +66341,7 @@ "azure/us/gpt-5.2": { "deprecation_date": "2027-06-08", "cache_read_input_token_cost": 1.925e-07, + "cache_read_input_token_cost_batches": 9.625e-08, "cache_read_input_token_cost_priority": 3.85e-07, "input_cost_per_token": 1.925e-06, "input_cost_per_token_batches": 9.625e-07, @@ -71596,15 +66353,6 @@ "output_cost_per_token_priority": 3.08e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/us/gpt-5.2-chat": { - "deprecation_date": "2026-06-29", - "cache_read_input_token_cost": 1.925e-07, - "input_cost_per_token": 1.925e-06, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 1.54e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/us/gpt-5.2-codex": { "deprecation_date": "2027-07-13", "cache_read_input_token_cost": 1.925e-07, @@ -71623,15 +66371,6 @@ "output_cost_per_token_batches": 9.24e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/us/gpt-5.3-chat": { - "deprecation_date": "2026-06-29", - "cache_read_input_token_cost": 1.925e-07, - "input_cost_per_token": 1.925e-06, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 1.54e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/us/gpt-5.3-codex": { "deprecation_date": "2027-08-24", "cache_read_input_token_cost": 1.925e-07, @@ -71647,6 +66386,7 @@ "azure/us/gpt-5.4-mini": { "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 8.25e-08, + "cache_read_input_token_cost_batches": 4.125e-08, "cache_read_input_token_cost_priority": 1.65e-07, "input_cost_per_token": 8.25e-07, "input_cost_per_token_batches": 4.125e-07, @@ -71661,6 +66401,7 @@ "azure/us/gpt-5.4-nano": { "deprecation_date": "2027-09-21", "cache_read_input_token_cost": 2.2e-08, + "cache_read_input_token_cost_batches": 1.1e-08, "input_cost_per_token": 2.2e-07, "input_cost_per_token_batches": 1.1e-07, "litellm_provider": "azure", @@ -71673,11 +66414,13 @@ "deprecation_date": "2027-09-07", "input_cost_per_token": 3.3e-05, "input_cost_per_token_above_272k_tokens": 6.6e-05, + "input_cost_per_token_above_272k_tokens_batches": 3.3e-05, "input_cost_per_token_batches": 1.65e-05, "litellm_provider": "azure", "mode": "chat", "output_cost_per_token": 0.000198, "output_cost_per_token_above_272k_tokens": 0.000297, + "output_cost_per_token_above_272k_tokens_batches": 0.0001485, "output_cost_per_token_batches": 9.9e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, @@ -71691,14 +66434,6 @@ "output_cost_per_token_batches": 2.42e-06, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" }, - "azure/us/o1-preview": { - "cache_read_input_token_cost": 8.25e-06, - "input_cost_per_token": 1.65e-05, - "litellm_provider": "azure", - "mode": "chat", - "output_cost_per_token": 6.6e-05, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" - }, "azure/us/o3-deep-research": { "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 2.75e-06, @@ -72854,6 +67589,16 @@ "supports_reasoning": true, "supports_vision": true }, + "openrouter/typesafe/jev-1.13": { + "input_cost_per_token": 4.2e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 32000, + "max_output_tokens": 28800, + "max_tokens": 28800, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://openrouter.ai/typesafe/jev-1.13" + }, "typesafe/jev-1.13.0": { "input_cost_per_token": 4.2e-08, "litellm_provider": "typesafe", @@ -72934,16 +67679,16 @@ "supports_web_search": true }, "openrouter/~anthropic/claude-opus-latest": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_1hr": 1e-05, - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 5e-06, + "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 8e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-06, "litellm_provider": "openrouter", "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 2.5e-05, + "output_cost_per_token": 2e-05, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -72978,15 +67723,14 @@ "supports_web_search": true }, "openrouter/~deepseek/deepseek-flash-latest": { - "cache_read_input_token_cost": 6e-09, - "input_cost_per_token": 3e-07, + "cache_read_input_token_cost": 3.6e-09, + "input_cost_per_token": 1.2e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, + "max_output_tokens": 943718, + "max_tokens": 943718, "mode": "chat", - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":1.5e-7,"output_cost_per_token":6e-7,"cache_read_input_token_cost":3e-9}, - "output_cost_per_token": 1.2e-06, + "output_cost_per_token": 4.8e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -72999,15 +67743,14 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-pro-latest": { - "cache_read_input_token_cost": 1.86208e-08, - "input_cost_per_token": 5.58624e-07, + "cache_read_input_token_cost": 1.2726e-08, + "input_cost_per_token": 3.9996e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 384000, - "max_tokens": 384000, + "max_output_tokens": 393216, + "max_tokens": 393216, "mode": "chat", - "off_peak_pricing": {"windows":[{"weekdays":["saturday","sunday"],"hours_utc":"00:00-00:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"00:00-01:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"04:00-06:00"},{"weekdays":["monday","tuesday","wednesday","thursday","friday"],"hours_utc":"10:00-00:00"}],"input_cost_per_token":5.58624e-7,"output_cost_per_token":0.000001675872,"cache_read_input_token_cost":1.86208e-8}, - "output_cost_per_token": 1.675872e-06, + "output_cost_per_token": 1.19988e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73020,14 +67763,14 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-v4-flash-latest": { - "cache_read_input_token_cost": 1.6e-08, - "input_cost_per_token": 4e-08, + "cache_read_input_token_cost": 8e-09, + "input_cost_per_token": 3e-08, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 943718, "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 6.4e-07, + "output_cost_per_token": 8e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73090,13 +67833,13 @@ }, "openrouter/~moonshotai/kimi-latest": { "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "input_cost_per_token": 1.4989e-06, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 943718, "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1.0758e-05, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73134,19 +67877,19 @@ "supports_web_search": true }, "openrouter/~openai/gpt-luna-latest": { - "cache_creation_input_token_cost": 2.5e-07, - "cache_creation_input_token_cost_above_272k_tokens": 5e-07, - "cache_read_input_token_cost": 2e-08, - "cache_read_input_token_cost_above_272k_tokens": 4e-08, - "input_cost_per_token": 2e-07, - "input_cost_per_token_above_272k_tokens": 4e-07, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, "litellm_provider": "openrouter", "max_input_tokens": 1050000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 1.2e-06, - "output_cost_per_token_above_272k_tokens": 1.8e-06, + "output_cost_per_token": 5e-07, + "output_cost_per_token_above_272k_tokens": 7.5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73252,14 +67995,14 @@ "supports_web_search": true }, "openrouter/~z-ai/glm-flash-latest": { - "cache_read_input_token_cost": 5e-08, - "input_cost_per_token": 1.5e-07, + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 7.5e-08, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 943718, "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 5e-07, + "output_cost_per_token": 2.5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73272,14 +68015,14 @@ "supports_web_search": false }, "openrouter/~z-ai/glm-latest": { - "cache_read_input_token_cost": 1.69e-07, - "input_cost_per_token": 9.1e-07, + "cache_read_input_token_cost": 1.2142e-07, + "input_cost_per_token": 6.538e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 2.86e-06, + "output_cost_per_token": 2.0548e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73295,7 +68038,7 @@ "cache_read_input_token_cost": 2e-07, "input_cost_per_token": 8e-07, "litellm_provider": "openrouter", - "max_input_tokens": 131072, + "max_input_tokens": 1048576, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -73315,7 +68058,7 @@ "cache_read_input_token_cost": 7.5e-07, "input_cost_per_token": 3e-06, "litellm_provider": "openrouter", - "max_input_tokens": 131072, + "max_input_tokens": 1048576, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -73335,7 +68078,7 @@ "cache_read_input_token_cost": 1.8e-07, "input_cost_per_token": 7e-07, "litellm_provider": "openrouter", - "max_input_tokens": 131072, + "max_input_tokens": 1048576, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -74034,87 +68777,8 @@ "supports_vision": false, "supports_web_search": false }, - "openrouter/deepseek/deepseek-v4-flash-0731:batch": { - "cache_read_input_token_cost": 3.5e-09, - "input_cost_per_token": 1.1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, - "mode": "chat", - "output_cost_per_token": 3.3e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, - "openrouter/deepseek/deepseek-v4-flash-0731:free": { - "input_cost_per_token": 0.0, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 393216, - "max_tokens": 393216, - "mode": "chat", - "output_cost_per_token": 0.0, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": false, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, - "openrouter/deepseek/deepseek-v4-flash-vision-exp:batch": { - "cache_read_input_token_cost": 3.5e-09, - "input_cost_per_token": 1.1e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, - "mode": "chat", - "output_cost_per_token": 3.3e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, - "openrouter/deepseek/deepseek-v4-pro-0813:batch": { - "cache_read_input_token_cost": 2.2e-08, - "input_cost_per_token": 6.6e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, - "mode": "chat", - "output_cost_per_token": 1.98e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/dots-studio/dots-3-note-preview:free": { - "deprecation_date": "2026-09-30", + "deprecation_date": "2026-12-31", "input_cost_per_token": 0.0, "litellm_provider": "openrouter", "max_input_tokens": 512000, @@ -74628,26 +69292,6 @@ "supports_vision": false, "supports_web_search": false }, - "openrouter/kwaipilot/kat-coder-pro-v2": { - "cache_read_input_token_cost": 6e-08, - "input_cost_per_token": 3e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 144000, - "max_tokens": 144000, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": false, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/kwaipilot/kat-coder-pro-v2.5": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 7.4e-07, @@ -74727,26 +69371,6 @@ "supports_vision": true, "supports_web_search": false }, - "openrouter/meta/muse-glimmer-30b:batch": { - "cache_read_input_token_cost": 2e-08, - "input_cost_per_token": 1.75e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 131072, - "max_output_tokens": 117964, - "max_tokens": 117964, - "mode": "chat", - "output_cost_per_token": 7.5e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, "openrouter/meta/muse-spark-1.1": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.25e-06, @@ -74885,26 +69509,6 @@ "supports_vision": false, "supports_web_search": false }, - "openrouter/minimax/minimax-m3:batch": { - "cache_read_input_token_cost": 6e-08, - "input_cost_per_token": 3e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 524288, - "max_output_tokens": 471859, - "max_tokens": 471859, - "mode": "chat", - "output_cost_per_token": 1.2e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, "openrouter/mistralai/codestral-2508:batch": { "cache_read_input_token_cost": 1.5e-08, "input_cost_per_token": 1.5e-07, @@ -75025,14 +69629,14 @@ "supports_web_search": false }, "openrouter/moonshotai/kimi-k3:batch": { - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 3e-06, + "cache_read_input_token_cost": 2.28e-07, + "input_cost_per_token": 2.28e-06, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 1.14e-05, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -75083,6 +69687,7 @@ "supports_web_search": false }, "openrouter/nex-agi/nex-n2.5-mini:free": { + "deprecation_date": "2026-09-25", "input_cost_per_token": 0.0, "litellm_provider": "openrouter", "max_input_tokens": 262144, @@ -75102,6 +69707,7 @@ "supports_web_search": false }, "openrouter/nex-agi/nex-n2.5-pro:free": { + "deprecation_date": "2026-09-25", "input_cost_per_token": 0.0, "litellm_provider": "openrouter", "max_input_tokens": 262144, @@ -75852,24 +70458,105 @@ "supports_vision": true, "supports_web_search": true }, - "openrouter/openai/gpt-oss-120b:batch": { - "input_cost_per_token": 1.5e-07, + "openrouter/openai/gpt-6-luna": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, "litellm_provider": "openrouter", - "max_input_tokens": 131072, - "max_output_tokens": 117964, - "max_tokens": 117964, + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 6e-07, + "output_cost_per_token": 5e-07, + "output_cost_per_token_above_272k_tokens": 7.5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": false, + "supports_pdf_input": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false + "supports_vision": true, + "supports_web_search": true + }, + "openrouter/openai/gpt-6-luna-pro": { + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, + "cache_read_input_token_cost": 1e-08, + "cache_read_input_token_cost_above_272k_tokens": 2e-08, + "input_cost_per_token": 1e-07, + "input_cost_per_token_above_272k_tokens": 2e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "output_cost_per_token_above_272k_tokens": 7.5e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "openrouter/openai/gpt-6-sol": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_272k_tokens": 1.5e-05, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "openrouter/openai/gpt-6-sol-pro": { + "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5e-06, + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "litellm_provider": "openrouter", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_above_272k_tokens": 1.5e-05, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true }, "openrouter/openai/o3-mini:batch": { "cache_read_input_token_cost": 2.75e-07, @@ -76045,45 +70732,6 @@ "supports_vision": true, "supports_web_search": true }, - "openrouter/qwen/qwen3.5-9b:batch": { - "input_cost_per_token": 1.7e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 262144, - "max_output_tokens": 235929, - "max_tokens": 235929, - "mode": "chat", - "output_cost_per_token": 2.5e-07, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": false, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, - "openrouter/qwen/qwen3.8-2.4t-a95b:batch": { - "cache_read_input_token_cost": 2.5e-07, - "input_cost_per_token": 2e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 1010000, - "max_output_tokens": 909000, - "max_tokens": 909000, - "mode": "chat", - "output_cost_per_token": 6e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/qwen/qwen3.8-27b:free": { "input_cost_per_token": 0.0, "litellm_provider": "openrouter", @@ -76616,26 +71264,6 @@ "supports_vision": true, "supports_web_search": false }, - "openrouter/thinkingmachines/inkling:batch": { - "cache_read_input_token_cost": 1.7e-07, - "input_cost_per_token": 1e-06, - "litellm_provider": "openrouter", - "max_input_tokens": 524288, - "max_output_tokens": 471859, - "max_tokens": 471859, - "mode": "chat", - "output_cost_per_token": 4.05e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": true, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": false, - "supports_tool_choice": true, - "supports_vision": true, - "supports_web_search": false - }, "openrouter/thinkingmachines/inkling:free": { "input_cost_per_token": 0.0, "litellm_provider": "openrouter", @@ -76757,35 +71385,15 @@ "supports_vision": true, "supports_web_search": true }, - "openrouter/z-ai/glm-5.2:batch": { - "cache_read_input_token_cost": 7e-08, - "input_cost_per_token": 7e-07, - "litellm_provider": "openrouter", - "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, - "mode": "chat", - "output_cost_per_token": 2.2e-06, - "source": "https://openrouter.ai/api/v1/models", - "supports_audio_input": false, - "supports_function_calling": true, - "supports_pdf_input": false, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": false, - "supports_web_search": false - }, "openrouter/z-ai/glm-5.3-flash:batch": { - "cache_read_input_token_cost": 1.5e-08, - "input_cost_per_token": 7.5e-08, + "cache_read_input_token_cost": 1.2e-08, + "input_cost_per_token": 6e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 2.5e-07, + "output_cost_per_token": 2e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -76798,14 +71406,14 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3:batch": { - "cache_read_input_token_cost": 1.3e-07, - "input_cost_per_token": 7e-07, + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 7.2e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 2.2e-06, + "output_cost_per_token": 2.4e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -77044,5 +71652,751 @@ "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true + }, + "xai/grok-4.20-0309": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-0309": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-latest-non-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-latest-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-non-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-beta-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-0304": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-0304-non-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-0304-reasoning": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-non-reasoning-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-experimental-beta-reasoning-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "xai/grok-4.20-multi-agent-beta-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "responses", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true, + "supported_endpoints": [ + "/v1/responses" + ] + }, + "xai/grok-4.20-multi-agent-experimental-beta-0304": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "responses", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true, + "supported_endpoints": [ + "/v1/responses" + ] + }, + "xai/grok-4.20-multi-agent-experimental-beta-latest": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "responses", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true, + "supported_endpoints": [ + "/v1/responses" + ] + }, + "xai/grok-4.20-non-reasoning-gv2": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_response_schema": true + }, + "xai/grok-4.20-reasoning-gv2": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "xai", + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, + "max_tokens": 1000000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true, + "input_cost_per_token_above_200k_tokens": 2.5e-06, + "output_cost_per_token_above_200k_tokens": 5e-06, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1.25e-06, + "supports_prompt_caching": true, + "supports_response_schema": true + }, + "openrouter/nex-agi/nex-n2.5-mini": { + "cache_read_input_token_cost": 2.5e-09, + "input_cost_per_token": 2.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "output_cost_per_token": 1e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": false, + "supports_pdf_input": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": false, + "supports_vision": true, + "supports_web_search": false + }, + "openrouter/nex-agi/nex-n2.5-pro": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 7.5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 235929, + "max_tokens": 235929, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": false + }, + "baseten/deepseek-ai/DeepSeek-V4.1-Flash": { + "cache_read_input_token_cost": 7e-09, + "input_cost_per_token": 3e-07, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/moonshotai/Kimi-K2.6": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "baseten", + "max_input_tokens": 262000, + "max_output_tokens": 262000, + "max_tokens": 262000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/moonshotai/Kimi-K2.7-Code": { + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "baseten", + "max_input_tokens": 262000, + "max_output_tokens": 262000, + "max_tokens": 262000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/moonshotai/Kimi-K3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { + "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "baseten", + "max_input_tokens": 202800, + "max_output_tokens": 202800, + "max_tokens": 202800, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "baseten/thinkingmachines/inkling": { + "cache_read_input_token_cost": 1.7e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 4.05e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/thinkingmachines/inkling-small": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 5e-07, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/zai-org/GLM-5.2": { + "cache_read_input_token_cost": 1.4e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/zai-org/GLM-5.3-Flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "baseten/deepseek-ai/DeepSeek-V4-Flash-0731": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.3e-07, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.6e-07, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "baseten/deepseek-ai/DeepSeek-V4-Pro": { + "cache_read_input_token_cost": 1.45e-07, + "input_cost_per_token": 1.74e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.48e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "baseten/deepseek-ai/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 1.32e-07, + "input_cost_per_token": 1.32e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "baseten/zai-org/GLM-5.2-Fast": { + "cache_read_input_token_cost": 2.1e-07, + "input_cost_per_token": 2.1e-06, + "litellm_provider": "baseten", + "max_input_tokens": 1048576, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 6.6e-06, + "source": "https://inference.baseten.co/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "xai/grok-code-fast": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "xai/grok-code-fast-1": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "xai/grok-code-fast-1-0825": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "input_cost_per_image_token": 1e-06, + "input_cost_per_token": 1e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "litellm_provider": "xai", + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "max_tokens": 256000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "output_cost_per_token_above_200k_tokens": 4e-06, + "source": "https://api.x.ai/v1/language-models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 0509516ac32..737a9b7fa60 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -113,6 +113,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "cache_creation_input_token_cost_above_272k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "cache_creation_input_token_cost_above_272k_tokens_flex": { "type": "number", "minimum": 0, @@ -123,6 +128,10 @@ "minimum": 0, "description": "Priority service-tier rate for the same-named base field." }, + "cache_creation_input_token_cost_batches": { + "type": "number", + "minimum": 0 + }, "cache_creation_input_token_cost_flex": { "type": "number", "minimum": 0, @@ -171,6 +180,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "cache_read_input_token_cost_above_272k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "cache_read_input_token_cost_above_272k_tokens_flex": { "type": "number", "minimum": 0, @@ -186,6 +200,10 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "cache_read_input_token_cost_batches": { + "type": "number", + "minimum": 0 + }, "cache_read_input_token_cost_flex": { "type": "number", "minimum": 0, @@ -338,6 +356,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "input_cost_per_token_above_272k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "input_cost_per_token_above_272k_tokens_flex": { "type": "number", "minimum": 0, @@ -586,6 +609,18 @@ "type": "number", "minimum": 0 }, + "output_cost_per_image_1024": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_image_1536": { + "type": "number", + "minimum": 0 + }, + "output_cost_per_image_512": { + "type": "number", + "minimum": 0 + }, "output_cost_per_image_token": { "type": "number", "minimum": 0 @@ -657,6 +692,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "output_cost_per_token_above_272k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "output_cost_per_token_above_272k_tokens_flex": { "type": "number", "minimum": 0, @@ -827,6 +867,9 @@ "supports_adaptive_thinking": { "type": "boolean" }, + "supports_anthropic_compaction": { + "type": "boolean" + }, "supports_anthropic_thinking_payload": { "type": "boolean" }, diff --git a/pyproject.toml b/pyproject.toml index a1b276e4e8c..f447343ff33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,11 +19,13 @@ dependencies = [ "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<3.0.0", "python-dotenv>=1.0.0,<2.0", + "pyyaml>=6.0.3,<7.0", + "packaging>=24.0", + "importlib-metadata>=8.0.0,<9.0", "tiktoken>=0.8.0,<1.0; python_version < '3.14'", "tiktoken>=0.12.0,<1.0; python_version >= '3.14'", - "importlib-metadata>=8.0.0,<9.0", - "packaging>=24.0", "tokenizers>=0.21.0,<1.0", + "huggingface-hub>=0.34.0,<2.0", "click>=8.0.0,<9.0", "jinja2>=3.1.6,<4.0", "aiohttp>=3.14.2,<4.0", @@ -191,6 +193,7 @@ litellm-proxy = "litellm.proxy.client.cli:litellm_proxy_cli" [dependency-groups] dev = [ + "numpy>=1.26.0,<3.0", "diff-cover==9.7.2", "hypothesis==6.165.10", "reportlab==5.0.1", @@ -290,6 +293,12 @@ healthcheck = [ "httpx==0.28.1", "pyyaml==6.0.3", ] +benchmarks = [ + "pytest==9.0.3", + "pytest-codspeed==4.3.0", + "mcp>=2.2.0,<3", + "a2a-sdk==1.1.0", +] [build-system] requires = ["maturin==1.15.0"] diff --git a/terraform/litellm/aws/locals.tf b/terraform/litellm/aws/locals.tf index 778d31642c1..b89bd486d02 100644 --- a/terraform/litellm/aws/locals.tf +++ b/terraform/litellm/aws/locals.tf @@ -74,7 +74,7 @@ locals { "/v1/models*", "/models*", "/openai/*", "/engines/*", "/v1/messages*", "/messages*", - "/v1/skills/*", "/v1/a2a/*", + "/v1/skills/*", "/v1/a2a/*", "/api/event_logging*", "/v1/rerank*", "/v2/rerank*", "/rerank*", "/v1/ocr*", "/ocr*", "/v1/rag/*", "/rag/*", diff --git a/terraform/litellm/gcp/locals.tf b/terraform/litellm/gcp/locals.tf index d263c781449..e82892b27cb 100644 --- a/terraform/litellm/gcp/locals.tf +++ b/terraform/litellm/gcp/locals.tf @@ -43,7 +43,7 @@ locals { "/v1/models*", "/models*", "/openai/*", "/engines/*", "/v1/messages*", "/messages*", - "/v1/skills/*", "/v1/a2a/*", + "/v1/skills/*", "/v1/a2a/*", "/api/event_logging*", "/v1/rerank*", "/v2/rerank*", "/rerank*", "/v1/ocr*", "/ocr*", "/v1/rag/*", "/rag/*", diff --git a/terraform/provider/CHANGELOG.md b/terraform/provider/CHANGELOG.md index ee5b42fe0b7..079bb7d8667 100644 --- a/terraform/provider/CHANGELOG.md +++ b/terraform/provider/CHANGELOG.md @@ -16,8 +16,18 @@ longer signal it. ### Added +- **key**: Computed `server_metadata` attribute on `litellm_key` exposing every metadata entry the proxy stores, so metadata created outside Terraform is visible in state and drift on it shows on refresh, while `metadata` keeps tracking only the declared entries and updates keep preserving undeclared ones - **team_member_add**: `tpm_limit`, `rpm_limit`, `budget_duration`, and `allowed_models` attributes on `litellm_team_member_add`, applied to every member of the resource; `budget_duration` and `allowed_models` ride on `/team/member_add`, while the limits are sent through `/team/member_update`, which is where the proxy accepts them - **team**: Optional `team_id` argument on `litellm_team`, so teams can be created with a stable, human-readable ID instead of a provider-generated UUID; changing it forces replacement +- `litellm_jwt_key_mapping` accepts `token_id` as an alternative to `key`, so a + mapping can name its virtual key by the SHA-256 hash the proxy stores instead + of by the plaintext. Exactly one of the two is required. This is what lets a + mapping reference a key managed in the same configuration + (`token_id = litellm_key.foo.token_id`), which `key` cannot do, because + `litellm_key` marks its generated key write-only and referencing it fails at + plan time. `POST /jwt/key/mapping/new` and `/jwt/key/mapping/update` gained a + matching `token` field, validated as 64 lowercase hex characters so a + plaintext key sent by mistake is rejected instead of hashed twice - **jwt_key_mapping**: New `litellm_jwt_key_mapping` resource for the proxy's JWT to virtual key mappings, so JWT clients identified by a claim (`client_id`, `azp`, `sub`) map to virtual keys and inherit their models, budgets and rate limits. Supports `description` and `is_active`, rotating the mapped key in place, and forces replacement when the claim name or value changes - **team**: `soft_budget`, `tags`, and `soft_budget_alerting_emails` attributes on `litellm_team`, matching what `/team/new` and `/team/update` already accept; `soft_budget_alerting_emails` is sent under `metadata`, where the proxy reads it - **user**: New `litellm_user` resource and `litellm_user` / `litellm_users` data sources for managing internal users diff --git a/terraform/provider/docs/resources/jwt_key_mapping.md b/terraform/provider/docs/resources/jwt_key_mapping.md index fbc30947113..726c4b16021 100644 --- a/terraform/provider/docs/resources/jwt_key_mapping.md +++ b/terraform/provider/docs/resources/jwt_key_mapping.md @@ -65,7 +65,8 @@ resource "litellm_jwt_key_mapping" "developer" { - `jwt_claim_name` - (Required, ForceNew) Name of the JWT claim to match on, for example `client_id`, `azp` or `sub`. Must match `virtual_key_claim_field` in the proxy JWT config - `jwt_claim_value` - (Required, ForceNew) Value of the claim identifying the JWT client. Unique together with `jwt_claim_name`, so a second mapping for the same pair fails with a 409 -- `key` - (Required, Sensitive) The virtual key this claim value maps to. It has to exist already, otherwise the proxy rejects the mapping with `The provided key does not match an existing virtual key` +- `key` - (Optional, Sensitive) The virtual key this claim value maps to, as plaintext. It has to exist already, otherwise the proxy rejects the mapping with `The provided key does not match an existing virtual key`. Exactly one of `key` or `token_id` is required. `litellm_key` marks its generated `key` write-only, so this cannot reference a `litellm_key` resource -- use `token_id` for that, or supply the plaintext from a variable or a secret manager +- `token_id` - (Optional) The SHA-256 hash of the virtual key this claim value maps to, which is what the proxy stores. `litellm_key` exposes it as `token_id`, so unlike `key` it can be referenced directly from a `litellm_key` resource. Not a secret, so it is not marked sensitive. Exactly one of `key` or `token_id` is required - `description` - (Optional) Description of the mapping - `is_active` - (Optional) Whether the mapping is active. Inactive mappings are ignored during JWT auth. Defaults to `true` diff --git a/terraform/provider/docs/resources/key.md b/terraform/provider/docs/resources/key.md index 0ef0688830f..af2f38b5352 100644 --- a/terraform/provider/docs/resources/key.md +++ b/terraform/provider/docs/resources/key.md @@ -120,6 +120,8 @@ In addition to all arguments above, the following attributes are exported: * `key` - The generated API key. This is the actual key value that will be used for authentication. +* `server_metadata` - Map of every metadata entry the proxy stores for this key, including entries not declared in `metadata`, so drift on them is visible on refresh. Entries already exposed as their own attributes (`model_rpm_limit`, `model_tpm_limit`, `tags`, `guardrails`, `enforced_params`, `allowed_passthrough_routes`, `rpm_limit_type`, `tpm_limit_type`, `prompts`) are omitted and non-string values are JSON encoded. Terraform never writes it; `metadata` still tracks only the entries declared in the configuration. + * `spend` - The current spend for this key. This reflects the total amount spent using this key so far. ## State Management diff --git a/terraform/provider/litellm/resource_jwt_key_mapping.go b/terraform/provider/litellm/resource_jwt_key_mapping.go index e606e865737..ea968821527 100644 --- a/terraform/provider/litellm/resource_jwt_key_mapping.go +++ b/terraform/provider/litellm/resource_jwt_key_mapping.go @@ -29,10 +29,17 @@ func resourceLiteLLMJWTKeyMapping() *schema.Resource { Description: "Value of the claim identifying the JWT client. Unique together with jwt_claim_name", }, "key": { - Type: schema.TypeString, - Required: true, - Sensitive: true, - Description: "The virtual key this claim value maps to. The proxy stores only a hash of it and never returns it, so drift on this attribute cannot be detected and Terraform tracks the configured value", + Type: schema.TypeString, + Optional: true, + Sensitive: true, + ExactlyOneOf: []string{"key", "token_id"}, + Description: "The virtual key this claim value maps to, as plaintext. The proxy stores only a hash of it and never returns it, so drift on this attribute cannot be detected and Terraform tracks the configured value. litellm_key marks its generated key write-only, so this cannot reference a litellm_key resource; use token_id for that, or supply the plaintext from a variable or a secret manager", + }, + "token_id": { + Type: schema.TypeString, + Optional: true, + ExactlyOneOf: []string{"key", "token_id"}, + Description: "The SHA-256 hash of the virtual key this claim value maps to, which is what the proxy stores. litellm_key exposes it as token_id, so unlike key it can be referenced directly from a litellm_key resource. Not a secret, so it is not marked sensitive", }, "description": { Type: schema.TypeString, diff --git a/terraform/provider/litellm/resource_jwt_key_mapping_crud.go b/terraform/provider/litellm/resource_jwt_key_mapping_crud.go index 725235305f6..6c3883de2e1 100644 --- a/terraform/provider/litellm/resource_jwt_key_mapping_crud.go +++ b/terraform/provider/litellm/resource_jwt_key_mapping_crud.go @@ -19,6 +19,7 @@ func resourceLiteLLMJWTKeyMappingCreate(d *schema.ResourceData, m interface{}) e JWTClaimName: d.Get("jwt_claim_name").(string), JWTClaimValue: d.Get("jwt_claim_value").(string), Key: d.Get("key").(string), + Token: d.Get("token_id").(string), Description: d.Get("description").(string), } @@ -95,6 +96,7 @@ func resourceLiteLLMJWTKeyMappingUpdate(d *schema.ResourceData, m interface{}) e client := m.(*Client) oldKey, _ := d.GetChange("key") + oldTokenID, _ := d.GetChange("token_id") oldDescription, _ := d.GetChange("description") oldIsActive, _ := d.GetChange("is_active") @@ -104,6 +106,7 @@ func resourceLiteLLMJWTKeyMappingUpdate(d *schema.ResourceData, m interface{}) e // attempting to resync, so a failed refresh can't leave the rejected // values persisted into state. d.Set("key", oldKey) + d.Set("token_id", oldTokenID) d.Set("description", oldDescription) d.Set("is_active", oldIsActive) if readErr := resourceLiteLLMJWTKeyMappingRead(d, m); readErr != nil { @@ -146,6 +149,7 @@ func updateJWTKeyMapping(d *schema.ResourceData, client *Client) error { updateRequest := JWTKeyMappingUpdateRequest{ ID: d.Id(), Key: d.Get("key").(string), + Token: d.Get("token_id").(string), Description: d.Get("description").(string), IsActive: d.Get("is_active").(bool), } diff --git a/terraform/provider/litellm/resource_jwt_key_mapping_crud_test.go b/terraform/provider/litellm/resource_jwt_key_mapping_crud_test.go index 8007d1d4e08..27b75849fe6 100644 --- a/terraform/provider/litellm/resource_jwt_key_mapping_crud_test.go +++ b/terraform/provider/litellm/resource_jwt_key_mapping_crud_test.go @@ -628,3 +628,99 @@ func TestJWTKeyMappingCreateDoesNotLeakKeyInErrors(t *testing.T) { t.Fatalf("the virtual key must be redacted in errors, got %v", err) } } + +func TestJWTKeyMappingCreateSendsTokenIDAndOmitsKey(t *testing.T) { + srv, calls := jwtKeyMappingTestServer(t, jwtKeyMappingFixture()) + defer srv.Close() + + const tokenHash = "1923314ae0efc8b2523c7d421bac5a7cf88df291273b139948b526d396974a41" + + client := NewClient(srv.URL, "test-key", true) + d := schema.TestResourceDataRaw(t, resourceLiteLLMJWTKeyMapping().Schema, map[string]interface{}{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice", + "token_id": tokenHash, + "is_active": true, + }) + + if err := resourceLiteLLMJWTKeyMappingCreate(d, client); err != nil { + t.Fatalf("create failed: %v", err) + } + + create := (*calls)[0] + if create.Body["token"] != tokenHash { + t.Fatalf("token hash not sent: %v", create.Body["token"]) + } + if _, sent := create.Body["key"]; sent { + t.Fatalf("key must be omitted when token_id is used, got: %v", create.Body) + } +} + +func TestJWTKeyMappingCreateOmitsTokenWhenKeyIsUsed(t *testing.T) { + srv, calls := jwtKeyMappingTestServer(t, jwtKeyMappingFixture()) + defer srv.Close() + + client := NewClient(srv.URL, "test-key", true) + d := schema.TestResourceDataRaw(t, resourceLiteLLMJWTKeyMapping().Schema, map[string]interface{}{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice", + "key": "sk-abc123", + "is_active": true, + }) + + if err := resourceLiteLLMJWTKeyMappingCreate(d, client); err != nil { + t.Fatalf("create failed: %v", err) + } + + create := (*calls)[0] + if create.Body["key"] != "sk-abc123" { + t.Fatalf("virtual key not sent: %v", create.Body["key"]) + } + if _, sent := create.Body["token"]; sent { + t.Fatalf("token must be omitted when key is used, got: %v", create.Body) + } +} + +func TestJWTKeyMappingUpdateSendsTokenIDAndOmitsKey(t *testing.T) { + srv, calls := jwtKeyMappingTestServer(t, jwtKeyMappingFixture()) + defer srv.Close() + + const oldHash = "1111111111111111111111111111111111111111111111111111111111111111" + const newHash = "2222222222222222222222222222222222222222222222222222222222222222" + + client := NewClient(srv.URL, "test-key", true) + d := resourceDataWithChange(t, + map[string]string{ + "id": "map-abc-123", + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice", + "token_id": oldHash, + "is_active": "true", + }, + map[string]interface{}{ + "jwt_claim_name": "client_id", + "jwt_claim_value": "dev-alice", + "token_id": newHash, + "is_active": true, + }) + + if err := resourceLiteLLMJWTKeyMappingUpdate(d, client); err != nil { + t.Fatalf("update failed: %v", err) + } + + var update *jwtKeyMappingCall + for i := range *calls { + if (*calls)[i].Path == "/jwt/key/mapping/update" { + update = &(*calls)[i] + } + } + if update == nil { + t.Fatalf("expected an update call, got %v", *calls) + } + if update.Body["token"] != newHash { + t.Fatalf("new token hash not sent: %v", update.Body["token"]) + } + if _, sent := update.Body["key"]; sent { + t.Fatalf("key must be omitted when token_id is used, got: %v", update.Body) + } +} diff --git a/terraform/provider/litellm/resource_key.go b/terraform/provider/litellm/resource_key.go index 39546d588df..b471cab73e8 100644 --- a/terraform/provider/litellm/resource_key.go +++ b/terraform/provider/litellm/resource_key.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "log" + "slices" "github.com/hashicorp/go-cty/cty" "github.com/hashicorp/terraform-plugin-sdk/v2/diag" @@ -61,6 +62,12 @@ func resourceKey() *schema.Resource { Optional: true, Elem: &schema.Schema{Type: schema.TypeString}, }, + "server_metadata": { + Type: schema.TypeMap, + Computed: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Description: "Every metadata entry the proxy stores for this key, including ones not declared in metadata. Read-only, so drift on undeclared entries shows up on refresh without Terraform taking ownership of them. Entries the provider already exposes as their own attributes (model_rpm_limit, model_tpm_limit, tags, guardrails, enforced_params, allowed_passthrough_routes, rpm_limit_type, tpm_limit_type, prompts) are omitted, and non-string values are JSON encoded", + }, "tpm_limit": { Type: schema.TypeInt, Optional: true, @@ -304,6 +311,7 @@ func resourceKeyRead(ctx context.Context, d *schema.ResourceData, m interface{}) return nil } + d.Set("server_metadata", serverKeyMetadata(key.Metadata)) key.Metadata = declaredKeyMetadata(key.Metadata, d.Get("metadata").(map[string]interface{})) mapKeyToResourceData(d, key) return nil @@ -395,6 +403,25 @@ func declaredKeyMetadata(server, declared map[string]interface{}) map[string]int return result } +func serverKeyMetadata(server map[string]interface{}) map[string]string { + result := make(map[string]string, len(server)) + for k, v := range server { + if slices.Contains(keyFieldsStoredInMetadata, k) { + continue + } + if s, ok := v.(string); ok { + result[k] = s + continue + } + encoded, err := json.Marshal(v) + if err != nil { + continue + } + result[k] = string(encoded) + } + return result +} + func mergeKeyMetadata(server, oldDeclared, newDeclared map[string]interface{}) map[string]interface{} { result := make(map[string]interface{}, len(server)+len(newDeclared)) for k, v := range server { diff --git a/terraform/provider/litellm/resource_key_test.go b/terraform/provider/litellm/resource_key_test.go index fe708edd3d3..b6e67360ad0 100644 --- a/terraform/provider/litellm/resource_key_test.go +++ b/terraform/provider/litellm/resource_key_test.go @@ -600,6 +600,12 @@ func TestKeyUpdateWithoutMetadataChangePreservesServerMetadata(t *testing.T) { if got := newState.Attributes["metadata.a"]; got != "1" { t.Errorf("metadata.a = %q, want 1", got) } + if got := newState.Attributes["server_metadata.server_side"]; got != "x" { + t.Errorf("server_metadata.server_side = %q, want x", got) + } + if _, present := proxy.updates[0]["server_metadata"]; present { + t.Errorf("computed server_metadata was sent on /key/update: %v", proxy.updates[0]["server_metadata"]) + } } func TestKeyUpdateWithMetadataChangeMergesOverServerMetadata(t *testing.T) { @@ -654,6 +660,32 @@ func TestKeyReadKeepsOnlyDeclaredMetadata(t *testing.T) { } } +func TestKeyReadExposesUndeclaredMetadataInServerMetadata(t *testing.T) { + proxy := &fakeKeyProxy{metadata: map[string]interface{}{ + "a": "1", + "server_side": "x", + "model_rpm_limit": map[string]interface{}{"gpt-4o-mini": float64(5)}, + "nested": map[string]interface{}{"k": "v"}, + }} + srv := httptest.NewServer(proxy.handler()) + defer srv.Close() + client := NewClient(srv.URL, "test-key", true) + + d := newKeyResourceData(t, map[string]interface{}{"metadata": map[string]interface{}{"a": "1"}}) + d.SetId("hash-1") + if diags := resourceKeyRead(context.Background(), d, client); diags.HasError() { + t.Fatalf("Read returned error: %v", diags) + } + + if got, want := d.Get("metadata"), map[string]interface{}{"a": "1"}; !reflect.DeepEqual(got, want) { + t.Errorf("metadata in state = %v, want %v", got, want) + } + want := map[string]interface{}{"a": "1", "server_side": "x", "nested": `{"k":"v"}`} + if got := d.Get("server_metadata"); !reflect.DeepEqual(got, want) { + t.Errorf("server_metadata in state = %v, want %v", got, want) + } +} + func TestKeyUpdateSendsChangedDuration(t *testing.T) { proxy := &fakeKeyProxy{metadata: map[string]interface{}{}} srv := httptest.NewServer(proxy.handler()) diff --git a/terraform/provider/litellm/types.go b/terraform/provider/litellm/types.go index 7bef44409fd..8bcf7dc4fe3 100644 --- a/terraform/provider/litellm/types.go +++ b/terraform/provider/litellm/types.go @@ -276,13 +276,15 @@ type VectorStoreInfoRequest struct { type JWTKeyMappingRequest struct { JWTClaimName string `json:"jwt_claim_name"` JWTClaimValue string `json:"jwt_claim_value"` - Key string `json:"key"` + Key string `json:"key,omitempty"` + Token string `json:"token,omitempty"` Description string `json:"description,omitempty"` } type JWTKeyMappingUpdateRequest struct { ID string `json:"id"` Key string `json:"key,omitempty"` + Token string `json:"token,omitempty"` Description string `json:"description"` IsActive bool `json:"is_active"` } diff --git a/tests/_process_helpers.py b/tests/_process_helpers.py new file mode 100644 index 00000000000..9f80c563fa9 --- /dev/null +++ b/tests/_process_helpers.py @@ -0,0 +1,57 @@ +"""Whether a killed process is really gone, for tests that kill whole process trees. + +A SIGKILLed grandchild whose parent died in the same ``killpg`` reparents to the +nearest subreaper or PID 1, and until that ancestor reaps it the pid is a zombie +that ``os.kill(pid, 0)`` still accepts. Reading its ``/proc`` state, and reaping +it when it landed on this process, keeps a runner that is slow to reap, or never +does, from turning a dead process into a failed assertion. The reap comes after +the liveness read so a child seen dying between the two is still collected on +the next poll instead of staying this process's own zombie. +""" + +import os +import time +from pathlib import Path +from typing import Final + +POLL_INTERVAL_S: Final = 0.05 + + +def _exists(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + return True + + +def _is_zombie(pid: int) -> bool: + try: + stat: Final = Path(f"/proc/{pid}/stat").read_text() + except OSError: + return False + return stat.rpartition(")")[2].split()[0] == "Z" + + +def _reap_if_ours(pid: int) -> None: + if os.name == "nt": + return + try: + os.waitpid(pid, os.WNOHANG) + except ChildProcessError: + pass + + +def _gone_now(pid: int) -> bool: + dead: Final = not _exists(pid) or _is_zombie(pid) + _reap_if_ours(pid) + return dead + + +def process_is_gone(pid: int, within_seconds: float) -> bool: + deadline: Final = time.monotonic() + within_seconds + while time.monotonic() < deadline: + if _gone_now(pid): + return True + time.sleep(POLL_INTERVAL_S) + return False diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 4d5a73779ea..ab046674eb6 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -8,6 +8,7 @@ from __future__ import annotations import ast import atexit import hashlib +import inspect import json import os import re @@ -15,10 +16,18 @@ import socket import sys import threading from collections import defaultdict -from typing import Iterable +from collections.abc import Iterable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final +from unittest import mock +import aiohttp import pytest +import vcr import vcr.matchers as _vcr_matchers +import vcr.patch as _vcr_patch from tests._vcr_redis_persister import ( MAX_EPISODES_PER_CASSETTE, @@ -127,9 +136,7 @@ def emit_vcr_diagnostic_log(terminalreporter) -> None: with open(path, "r", encoding="utf-8") as fh: content = fh.read() except OSError as exc: - read_errors.append( - f" [failed to read {name}: {type(exc).__name__}: {exc}]" - ) + read_errors.append(f" [failed to read {name}: {type(exc).__name__}: {exc}]") continue for line in content.splitlines(): if not line.strip(): @@ -142,9 +149,7 @@ def emit_vcr_diagnostic_log(terminalreporter) -> None: return terminalreporter.write_sep("=", "VCR DIAGNOSTIC LOG", bold=True) - terminalreporter.write_line( - f" source dir: {directory} (deduplicated; full log archived as a CI artifact)" - ) + terminalreporter.write_line(f" source dir: {directory} (deduplicated; full log archived as a CI artifact)") for line in read_errors: terminalreporter.write_line(line) @@ -235,9 +240,7 @@ def pin_httpx_multipart_boundary(monkeypatch) -> None: boundary = VCR_FIXED_MULTIPART_BOUNDARY.encode("ascii") return _original_init(self, data=data, files=files, boundary=boundary, **kwargs) - monkeypatch.setattr( - _httpx_multipart.MultipartStream, "__init__", _init_with_fixed_boundary - ) + monkeypatch.setattr(_httpx_multipart.MultipartStream, "__init__", _init_with_fixed_boundary) @pytest.fixture(scope="session", autouse=True) @@ -270,11 +273,7 @@ def _replace_b64_json_in_place(obj) -> bool: changed = False if isinstance(obj, dict): for key, value in obj.items(): - if ( - key == "b64_json" - and isinstance(value, str) - and len(value) > len(VCR_IMAGE_B64_PLACEHOLDER) - ): + if key == "b64_json" and isinstance(value, str) and len(value) > len(VCR_IMAGE_B64_PLACEHOLDER): obj[key] = VCR_IMAGE_B64_PLACEHOLDER changed = True elif _replace_b64_json_in_place(value): @@ -296,16 +295,12 @@ def _strip_image_b64_payloads(response): preserves all those checks while shrinking cassettes by ~99%. """ if not isinstance(response, dict): - vcr_diag_write_line( - f"[vcr-strip-b64] response is {type(response).__name__!r}, not " - "dict; skipping b64 scrub" - ) + vcr_diag_write_line(f"[vcr-strip-b64] response is {type(response).__name__!r}, not dict; skipping b64 scrub") return response body = response.get("body") if not isinstance(body, dict): vcr_diag_write_line( - f"[vcr-strip-b64] response['body'] is {type(body).__name__!r}, " - "not dict; skipping b64 scrub" + f"[vcr-strip-b64] response['body'] is {type(body).__name__!r}, not dict; skipping b64 scrub" ) return response raw = body.get("string") @@ -316,10 +311,7 @@ def _strip_image_b64_payloads(response): try: text = bytes(raw).decode("utf-8") except UnicodeDecodeError: - vcr_diag_write_line( - "[vcr-strip-b64] response body bytes are not valid UTF-8; " - "skipping b64 scrub" - ) + vcr_diag_write_line("[vcr-strip-b64] response body bytes are not valid UTF-8; skipping b64 scrub") return response was_bytes = True elif isinstance(raw, str): @@ -327,8 +319,7 @@ def _strip_image_b64_payloads(response): was_bytes = False else: vcr_diag_write_line( - f"[vcr-strip-b64] response['body']['string'] is " - f"{type(raw).__name__!r}, not bytes/str; skipping b64 scrub" + f"[vcr-strip-b64] response['body']['string'] is {type(raw).__name__!r}, not bytes/str; skipping b64 scrub" ) return response @@ -349,9 +340,7 @@ def _strip_image_b64_payloads(response): for key in list(headers): if str(key).lower() == "content-length": value = headers[key] - headers[key] = ( - [new_len_value] if isinstance(value, list) else new_len_value - ) + headers[key] = [new_len_value] if isinstance(value, list) else new_len_value return response @@ -409,15 +398,11 @@ def _canonical_body(request) -> tuple[bytes, str]: # selected. This mirrors the existing SigV4 / multipart-boundary / b64-image # normalizations already in this module, and means the already-bloated # cassettes start replaying immediately without a flush + re-record. -_VCR_UUID_RE = re.compile( - rb"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" -) +_VCR_UUID_RE = re.compile(rb"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") _VCR_LITELLM_BATCH_JOB_RE = re.compile(rb"litellm-batch-[0-9a-fA-F]{8}") # ISO-8601 timestamps, e.g. ``2026-05-25T03:40:37.262045Z`` / # ``2026-05-25T03:40:37+00:00``. -_VCR_ISO_TS_RE = re.compile( - rb"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?" -) +_VCR_ISO_TS_RE = re.compile(rb"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?") # Unix epoch as 13-digit milliseconds, then 10-digit ``time.time()`` float, # then 10-digit integer seconds. Anchored to ``1`` + 9/12 digits, which keeps # them inside the 2001-2033 / 2001-2033 epoch windows and avoids matching @@ -639,10 +624,7 @@ def _should_drop_telemetry_record(request) -> bool: return False if not _is_telemetry_request(request): return False - if ( - _is_telemetry_export_request(request) - and not _current_test_replays_telemetry_export() - ): + if _is_telemetry_export_request(request) and not _current_test_replays_telemetry_export(): return True return not _current_test_records_telemetry() @@ -767,9 +749,7 @@ def _iter_header_values(headers, name: str): yield value -_AWS_SIGV4_CREDENTIAL_RE = re.compile( - r"AWS4-HMAC-SHA256\s+Credential=([^/\s,]+)/", re.IGNORECASE -) +_AWS_SIGV4_CREDENTIAL_RE = re.compile(r"AWS4-HMAC-SHA256\s+Credential=([^/\s,]+)/", re.IGNORECASE) # Google OAuth2 access tokens always start with ``ya29.`` regardless of how # they were minted (service account, metadata server, impersonation). @@ -891,9 +871,7 @@ def _normalize_multipart_boundary(request) -> None: return try: - headers[content_type_key] = content_type_value.replace( - match.group(0), fixed_param - ) + headers[content_type_key] = content_type_value.replace(match.group(0), fixed_param) except (TypeError, AttributeError): return @@ -985,8 +963,7 @@ def _materialize_iterable_body(request) -> None: uri = getattr(request, "uri", getattr(request, "url", "?")) first_type = type(chunks[0]).__name__ if chunks else "empty" vcr_diag_write_line( - f"[vcr-materialize] FALLBACK: {method} {uri} chunk type " - f"{first_type!r} not coerced to bytes; storing b''" + f"[vcr-materialize] FALLBACK: {method} {uri} chunk type {first_type!r} not coerced to bytes; storing b''" ) out = b"" @@ -1026,9 +1003,7 @@ def _key_fingerprint_matcher(r1, r2) -> None: return def _fp(req): - for value in _iter_header_values( - getattr(req, "headers", None), KEY_FINGERPRINT_HEADER - ): + for value in _iter_header_values(getattr(req, "headers", None), KEY_FINGERPRINT_HEADER): if value is None: continue return value if isinstance(value, str) else str(value) @@ -1159,13 +1134,11 @@ def _print_atexit_banner() -> None: _emit("VCR CASSETTE CACHE DEGRADED") if save_failures: _emit( - f" {save_failures} cassette save failure(s); last error: " - f"{health.get('save_failure_last_error', '')}" + f" {save_failures} cassette save failure(s); last error: {health.get('save_failure_last_error', '')}" ) if load_failures: _emit( - f" {load_failures} cassette load failure(s); last error: " - f"{health.get('load_failure_last_error', '')}" + f" {load_failures} cassette load failure(s); last error: {health.get('load_failure_last_error', '')}" ) if snapshot: _emit(_format_capacity_line(snapshot)) @@ -1276,11 +1249,7 @@ class _RespxUsageVisitor(ast.NodeVisitor): if isinstance(dec, ast.Call): dec = dec.func if isinstance(dec, ast.Attribute): - return ( - isinstance(dec.value, ast.Name) - and dec.value.id == "respx" - and dec.attr == "mock" - ) + return isinstance(dec.value, ast.Name) and dec.value.id == "respx" and dec.attr == "mock" return False def _is_pytest_mark_respx(self, dec: ast.expr) -> bool: @@ -1307,9 +1276,7 @@ class _RespxUsageVisitor(ast.NodeVisitor): # ``def test_foo(respx_mock): ...`` — pytest supplies the fixture # whenever the parameter name appears, regardless of marker. all_args = ( - list(args.args) - + list(args.kwonlyargs) - + (list(args.posonlyargs) if hasattr(args, "posonlyargs") else []) + list(args.args) + list(args.kwonlyargs) + (list(args.posonlyargs) if hasattr(args, "posonlyargs") else []) ) for a in all_args: if a.arg == "respx_mock": @@ -1566,9 +1533,7 @@ def _emit_outcome_payload( }, ) ) - node.user_properties.append( - (_USER_PROP_RECORDED_BY, os.environ.get("PYTEST_XDIST_WORKER", "")) - ) + node.user_properties.append((_USER_PROP_RECORDED_BY, os.environ.get("PYTEST_XDIST_WORKER", ""))) def aggregate_report_outcome(report) -> None: @@ -1616,9 +1581,7 @@ def aggregate_report_outcome(report) -> None: if verdict == VERDICT_MISS_OVERFLOW: _session_stats["overflow_tests"].append(nodeid) elif verdict == VERDICT_UNMARKED_LIVE_CALL: - _session_stats["unmarked_live_call_tests"].append( - (nodeid, list(outcome.get("live_call_hosts") or [])) - ) + _session_stats["unmarked_live_call_tests"].append((nodeid, list(outcome.get("live_call_hosts") or []))) skip_reason = outcome.get("skip_reason") if skip_reason: @@ -1635,9 +1598,7 @@ def session_stats_snapshot() -> dict: "overflow_tests": list(_session_stats["overflow_tests"]), "unmarked_live_call_tests": list(_session_stats["unmarked_live_call_tests"]), "skip_reason_counts": dict(_session_stats["skip_reason_counts"]), - "skip_reason_examples": { - k: list(v) for k, v in _session_stats["skip_reason_examples"].items() - }, + "skip_reason_examples": {k: list(v) for k, v in _session_stats["skip_reason_examples"].items()}, } @@ -1810,9 +1771,7 @@ def record_vcr_outcome(request, vcr) -> None: # Cassette is None ⇒ test wasn't VCR-marked. Honor the skip reason # we tagged at collection time, and pull live-call hosts captured by # the socket probe (if any). - skip_reason = getattr( - request.node, VCR_SKIP_REASON_USER_ATTR, SKIP_REASON_FILE_OPT_OUT - ) + skip_reason = getattr(request.node, VCR_SKIP_REASON_USER_ATTR, SKIP_REASON_FILE_OPT_OUT) _session_stats["skip_reason_counts"][skip_reason] += 1 hosts = getattr(request.node, _LIVE_CALL_BUFFER_KEY, []) or [] @@ -1837,9 +1796,7 @@ def record_vcr_outcome(request, vcr) -> None: live_call_hosts=hosts, ) if vcr_outcome_logging_enabled(): - request.node.user_properties.append( - (_USER_PROP_VERDICT_LINE, _format_verdict_line(verdict, None, extra)) - ) + request.node.user_properties.append((_USER_PROP_VERDICT_LINE, _format_verdict_line(verdict, None, extra))) def install_live_call_probe(request, vcr) -> None: @@ -1858,9 +1815,7 @@ def install_live_call_probe(request, vcr) -> None: # Track the current test for telemetry-leak suppression (applies to every # test, VCR-marked or not). See ``_should_drop_telemetry_record``. global _current_test_nodeid - _current_test_nodeid = str( - getattr(getattr(request, "node", None), "nodeid", "") or "" - ) + _current_test_nodeid = str(getattr(getattr(request, "node", None), "nodeid", "") or "") if vcr is not None or vcr_disabled(): return None probe = _LiveCallProbe() @@ -1876,10 +1831,7 @@ def _format_capacity_line(snapshot: dict) -> str: pct = float(snapshot.get("used_pct", 0.0) or 0.0) used_mb = used / (1024 * 1024) cap_mb = cap / (1024 * 1024) - return ( - f" Cassette Redis usage: {used_mb:.1f} MiB / {cap_mb:.1f} MiB " - f"({pct:.1f}% of maxmemory)" - ) + return f" Cassette Redis usage: {used_mb:.1f} MiB / {cap_mb:.1f} MiB ({pct:.1f}% of maxmemory)" def emit_vcr_classification_summary(terminalreporter) -> None: @@ -1940,14 +1892,10 @@ def emit_vcr_classification_summary(terminalreporter) -> None: total_leaks = sum(leak_counts.values()) terminalreporter.write_sep("-", "VCR COST LEAK CHECK", bold=True) if total_leaks: - rendered = ", ".join( - f"{verdict}={count}" for verdict, count in leak_counts.items() if count - ) + rendered = ", ".join(f"{verdict}={count}" for verdict, count in leak_counts.items() if count) terminalreporter.write_line(f" FAIL: {rendered}") else: - terminalreporter.write_line( - " PASS: no overflow, partial, not-persisted, or unmarked live-call verdicts" - ) + terminalreporter.write_line(" PASS: no overflow, partial, not-persisted, or unmarked live-call verdicts") overflow = snapshot["overflow_tests"] if overflow: @@ -2007,18 +1955,14 @@ def emit_cassette_cache_session_banner(terminalreporter) -> None: snapshot = cassette_cache_capacity_snapshot() if save_failures or load_failures: - terminalreporter.write_sep( - "=", "VCR CASSETTE CACHE DEGRADED", red=True, bold=True - ) + terminalreporter.write_sep("=", "VCR CASSETTE CACHE DEGRADED", red=True, bold=True) if save_failures: terminalreporter.write_line( - f" {save_failures} cassette save failure(s); last error: " - f"{health.get('save_failure_last_error', '')}" + f" {save_failures} cassette save failure(s); last error: {health.get('save_failure_last_error', '')}" ) if load_failures: terminalreporter.write_line( - f" {load_failures} cassette load failure(s); last error: " - f"{health.get('load_failure_last_error', '')}" + f" {load_failures} cassette load failure(s); last error: {health.get('load_failure_last_error', '')}" ) terminalreporter.write_line( " Tests still passed because cassette persistence is best-effort, " @@ -2031,9 +1975,7 @@ def emit_cassette_cache_session_banner(terminalreporter) -> None: return if snapshot and snapshot["used_pct"] >= CASSETTE_CACHE_HIGH_WATER_FRACTION * 100: - terminalreporter.write_sep( - "=", "VCR CASSETTE CACHE NEAR CAPACITY", yellow=True, bold=True - ) + terminalreporter.write_sep("=", "VCR CASSETTE CACHE NEAR CAPACITY", yellow=True, bold=True) terminalreporter.write_line(_format_capacity_line(snapshot)) terminalreporter.write_line( " No save failures yet, but Redis is approaching maxmemory. " @@ -2082,13 +2024,104 @@ class VerboseReporterState: if reporter is None: return verdict = next( - ( - v - for k, v in (report.user_properties or []) - if k == _USER_PROP_VERDICT_LINE - ), + (v for k, v in (report.user_properties or []) if k == _USER_PROP_VERDICT_LINE), None, ) if not verdict: return reporter.write_line(f"{verdict} :: {report.nodeid}") + + +@dataclass(frozen=True, slots=True) +class VcrPatchPoint: + owner: object + attribute: str + original: object + + @property + def name(self) -> str: + return f"{_patch_owner_name(self.owner)}.{self.attribute}" + + def current(self) -> object: + current: Final[object] = getattr(self.owner, self.attribute) + return current + + def is_patched(self) -> bool: + return self.current() is not self.original + + def restore(self) -> None: + setattr(self.owner, self.attribute, self.original) + + +def _patch_owner_name(owner: object) -> str: + if inspect.isclass(owner): + return f"{owner.__module__}.{owner.__qualname__}" + if inspect.ismodule(owner): + return owner.__name__ + return repr(owner) + + +def _vcr_patch_point(patcher: mock._patch[object]) -> VcrPatchPoint: + owner: Final[object] = patcher.getter() + return VcrPatchPoint(owner=owner, attribute=patcher.attribute, original=patcher.new) + + +_VCR_PATCH_POINTS: Final = ( + *(_vcr_patch_point(patcher) for patcher in _vcr_patch.reset_patchers()), + VcrPatchPoint(aiohttp.ClientSession, "_request", _vcr_patch._AiohttpClientSessionRequest), +) + + +@dataclass(frozen=True, slots=True) +class VcrPatchLeak: + patch_points: tuple[str, ...] + cassette_paths: tuple[str, ...] + + +def _cassette_paths_wrapped_into(fn: object) -> tuple[str, ...]: + if not inspect.isfunction(fn): + return () + cassette: Final = inspect.getclosurevars(fn).nonlocals.get("cassette") + own: Final = (str(cassette._path),) if isinstance(cassette, vcr.cassette.Cassette) else () + return own + _cassette_paths_wrapped_into(getattr(fn, "__wrapped__", None)) + + +def detect_vcr_patch_leak() -> VcrPatchLeak | None: + leaked: Final = tuple(point for point in _VCR_PATCH_POINTS if point.is_patched()) + if not leaked: + return None + return VcrPatchLeak( + patch_points=tuple(point.name for point in leaked), + cassette_paths=tuple( + dict.fromkeys(path for point in leaked for path in _cassette_paths_wrapped_into(point.current())) + ), + ) + + +def restore_vcr_patch_points() -> None: + for point in _VCR_PATCH_POINTS: + point.restore() + + +def guard_vcr_patch_points(item: pytest.Item, teardown_failed: bool) -> None: + leak: Final = detect_vcr_patch_leak() + if leak is None: + return + restore_vcr_patch_points() + if teardown_failed: + return + pytest.fail( + f"{item.nodeid} finished with a vcrpy cassette still patched into " + f"{', '.join(leak.patch_points)} (cassettes: {', '.join(leak.cassette_paths) or 'unknown'}); " + "the originals were restored so later tests are unaffected", + pytrace=False, + ) + + +@contextmanager +def rewound_new_episodes_cassette(cassette_dir: Path) -> Iterator[vcr.cassette.Cassette]: + cassette_path: Final = cassette_dir / "rewound_owner.yaml" + cassette_path.write_text("interactions: []\nversion: 1\n") + recorder: Final = vcr.VCR(cassette_library_dir=str(cassette_dir)) + with recorder.use_cassette(cassette_path.name, record_mode="new_episodes") as cassette: + yield cassette diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index c9b31cfb7d7..309ecab9991 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -8,8 +8,11 @@ flipping results between runs. Running the executor inline keeps each benchmark's cost self-contained and deterministic. """ +import os +import sys from collections.abc import Callable, Iterator from concurrent.futures import Future +from pathlib import Path from typing import ParamSpec, TypeVar import pytest @@ -20,6 +23,21 @@ P = ParamSpec("P") R = TypeVar("R") +def pytest_configure(config: pytest.Config) -> None: + if os.environ.get("LITELLM_REQUIRE_INSTALLED_WHEEL") != "1": + return + + import litellm + import litellm.rust_bridge._native as native + + prefix = Path(sys.prefix).resolve() + for name, module_file in (("litellm", litellm.__file__), ("litellm.rust_bridge._native", native.__file__)): + path = Path(module_file).resolve() + if not path.is_relative_to(prefix): + raise pytest.UsageError(f"{name} resolved outside the benchmark environment: {path}") + print(f"{name}: {path}") # noqa: T201 # provenance evidence must be visible in CI logs + + def _submit_inline(fn: Callable[P, R], /, *args: P.args, **kwargs: P.kwargs) -> Future[R]: future: Future[R] = Future() try: diff --git a/tests/capturing_transport.py b/tests/capturing_transport.py new file mode 100644 index 00000000000..496c286685c --- /dev/null +++ b/tests/capturing_transport.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final + +import httpx +from pydantic import BaseModel, TypeAdapter + +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) + + +class CapturingTransport(httpx.AsyncBaseTransport, httpx.BaseTransport): + def __init__(self, response: BaseModel) -> None: + self._response: Final = response + self.request_bodies: tuple[Mapping[str, object], ...] = () + + def handle_request(self, request: httpx.Request) -> httpx.Response: + return self._respond(request.read()) + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + return self._respond(await request.aread()) + + def _respond(self, body: bytes) -> httpx.Response: + self.request_bodies = (*self.request_bodies, _JSON_OBJECT.validate_json(body)) + return httpx.Response(200, json=self._response.model_dump(mode="json")) diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 071191183df..b77348b5d73 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -67,6 +67,7 @@ IGNORE_FUNCTIONS = [ "_redact_agent_params_tree", # max depth set (default 10), same shape as _redact_sensitive_litellm_params. "_restore_redacted_nested_value", # max depth set (default 10), mirrors _redact_agent_params_tree on the write side. "_unqualified", # bounded by the qualifier depth of a static TypedDict annotation (Annotated, Required/NotRequired, ReadOnly around one type, no cycles possible). + "_render_json", # bounded by the nesting depth of a pydantic-validated JsonValue from the operator's config (a finite JSON tree, no cycles possible). "completion_cost", # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None. ] diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py index 78e6562a4a8..758d579f67d 100644 --- a/tests/code_coverage_tests/test_e2e_changed_gate.py +++ b/tests/code_coverage_tests/test_e2e_changed_gate.py @@ -106,10 +106,11 @@ def test_short_values_are_written_without_masking_every_digit_in_the_log(tmp_pat env_path: Final = tmp_path / ".env" result: Final = subprocess.run( - [sys.executable, str(SECRETS_TO_ENV), str(env_path)], + [sys.executable, "-I", str(SECRETS_TO_ENV), str(env_path)], input='{"FLAG": "1", "API_KEY": "sk-0123456789abcdef"}', capture_output=True, text=True, + env={**os.environ, "GITHUB_ACTIONS": "true"}, ) assert result.returncode == 0, result.stderr @@ -117,6 +118,24 @@ def test_short_values_are_written_without_masking_every_digit_in_the_log(tmp_pat assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n" +def test_outside_actions_no_value_is_printed(tmp_path: Path) -> None: + env_path: Final = tmp_path / ".env" + local_env: Final = {key: value for key, value in os.environ.items() if key != "GITHUB_ACTIONS"} + + result: Final = subprocess.run( + [sys.executable, "-I", str(SECRETS_TO_ENV), str(env_path)], + input='{"FLAG": "1", "API_KEY": "sk-0123456789abcdef"}', + capture_output=True, + text=True, + env=local_env, + ) + + assert result.returncode == 0, result.stderr + assert result.stdout == "" + assert "sk-0123456789abcdef" not in result.stderr + assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n" + + def redact_output(tmp_path: Path, values: tuple[str, ...], text: str) -> tuple[subprocess.CompletedProcess[str], Path]: env_path: Final = tmp_path / ".env" _ = env_path.write_text("".join(f"{name}='{value}'\n" for name, value in zip(("A", "B", "C"), values))) @@ -212,6 +231,11 @@ def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]: (("tests/e2e/batches/test_managed_files_enforcement_e2e.py",), ()), (("tests/e2e/guardrails/test_presidio_masking_e2e.py",), ()), (("tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py",), ()), + (("tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py",), ()), + ( + ("tests/e2e/logging/test_team_langfuse_callback_e2e.py",), + ("tests/e2e/logging/test_team_langfuse_callback_e2e.py",), + ), ( ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",), ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",), diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index b00b7dfac95..d948c6fd1d9 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -43,7 +43,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `mcp/` - the MCP server surface over api_key auth against the real Datadog remote MCP server (see "MCP suite: real Datadog only" below); plus the gateway-managed OAuth (authorization_code) path exercised through `/chat/completions` in `test_mcp_chat_completion_oauth_e2e.py` and direct MCP protocol operations in `test_mcp_oauth_happy_path_e2e.py`, the one behavior Datadog's static-header auth cannot reach, seeding the per-user upstream token via the interactive authorize dance driven with the mcp SDK's own OAuth client (headless-browser consent from a saved session) and asserting the completion or protocol call lists and executes the server's tools with the stored per-user token - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection -- `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory regression test (`test_reliability_memory_e2e.py`: a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak) +- `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory tests (`test_reliability_memory_e2e.py`: every worker's RSS as read at collection time, before any test traffic, must sit under a fixed idle budget, the release-gate check for a DB-backed boot that idles near the pod limit the way v1.100.x did; and a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak) - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests @@ -97,7 +97,7 @@ Each suite provides its own `client` fixture (see `llm_translation/passthrough_c Request and response bodies are typed pydantic models in `models.py`; only the fields a test reads are modelled, and nothing passes raw dicts. Outcomes come back as a `Result[R]` tagged union (`Success`, `NetworkError`, `UnauthorizedError`, `RateLimitedError`, `ValidationError`, `UnknownApiError`). Handle them with `match`, or call `unwrap(...)` when a non-success should fail the test. The harness hard-fails and never skips: a test marked `e2e` fails when no proxy answers its liveness probe, and once a request reaches the proxy any wrong behavior is likewise a hard failure, so a missing proxy turns the run red instead of being mistaken for a pass -Mark live tests with `@pytest.mark.e2e` (on the class or the module). Use `scoped_key` for a fresh all-models key that auto-deletes, `resources` when you need to create and tear down more than a key, and `unique_marker()` from `e2e_config` to keep prompts, tags, and customer ids from colliding across concurrent runs and the shared response cache +Mark live tests with `@pytest.mark.e2e` (on the class or the module). Add `@pytest.mark.quiet_stack` to a test that measures the proxy itself (RSS, latency): the shared stack lock in `stack_lock.py` then runs it while no other test on the host is hitting the stack, marked or not, so the reading depends only on the test's own traffic. Use `scoped_key` for a fresh all-models key that auto-deletes, `resources` when you need to create and tear down more than a key, and `unique_marker()` from `e2e_config` to keep prompts, tags, and customer ids from colliding across concurrent runs and the shared response cache ## Record and replay fixtures @@ -191,7 +191,7 @@ reliability... behavior : fallback | retry | cooldown | timeout | routing | cache | circuit_breaker | perf variant : 5xx | context_window | content_policy | 429 | timeout simple_shuffle | usage_based | latency_based | cost_based | least_busy - latency | throughput | session_anomaly | memory (perf only; SLO/threshold assertion, not binary) + latency | throughput | session_anomaly | memory | idle_memory (perf only; SLO/threshold assertion, not binary) assertion : routes_to_fallback | succeeds_within_retries | picks_under_tpm | returns_cached | trips_then_recovers | under_slo e.g. reliability.fallback.context_window.routes_to_fallback exercised_on=[chat_completions] @@ -210,14 +210,15 @@ quota_management... chat_completions | stream | messages_bridge | embeddings | cache_hit | key_rollup | concurrent_burst | tags | end_user | per_model | failure | spend_calculate | pagination | key_attribution + | websearch_interception assertion : blocks_over_limit | resets_after_window | headers_report_remaining | picks_under_tpm | blocks_then_resets | resets_windows_independently | alerts_without_blocking | isolates_per_model | isolates_per_member | isolates_per_group | enforced_across_keys | routes_to_fallback | reseed_matches_db | reports_spend | logs_cost | zero_cost | matches_sum_of_logs | loses_no_spend | attributes_spend | writes_own_rows - | writes_failure_row | returns_cost | keeps_total | joins_key | reports_alias_and_email + | writes_failure_row | attributes_provider | returns_cost | keeps_total | joins_key | reports_alias_and_email | health_rows_keep_service_account | retrieve_batch_cost_joins_retrieving_key - | poller_batch_cost_joins_creating_key + | poller_batch_cost_joins_creating_key | bills_under_request_session e.g. quota_management.ratelimit.rpm.blocks_over_limit exercised_on=[chat_completions, messages] quota_management.budget.key.blocks_over_limit exercised_on=[chat_completions] ``` diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 2b59e8770b5..15c2d6763d6 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -105,7 +105,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. `logging/test_otel_v2_langfuse_generation_output_e2e.py` is marked `otel_v2` and deselects itself unless `E2E_OTEL_V2` is set, because it needs a gateway booted with `LITELLM_OTEL_V2=true` and Langfuse credentials, neither of which this stack provides, so run it with `E2E_OTEL_V2=1` against a local OTel v2 proxy. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/batches/COVERAGE.md b/tests/e2e/batches/COVERAGE.md index d18bed6c088..1731b4c620d 100644 --- a/tests/e2e/batches/COVERAGE.md +++ b/tests/e2e/batches/COVERAGE.md @@ -21,6 +21,7 @@ failures are hard test failures (see `tests/e2e/AGENTS.md`). | Vertex AI | yes | yes | yes | yes | yes (provider-transformed) | GCS (`gcs_bucket_name` / `GCS_BUCKET_NAME` on model) | | Bedrock | yes (unified only) | yes | yes | yes (unfiltered managed list) | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` on model) | | Bedrock GovCloud (`us-gov-west-1`) | yes (unified only) | yes | no | no | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` on model, resolved from `AWS_GOVCLOUD_ACCESS_KEY_ID` / `AWS_GOVCLOUD_SECRET_ACCESS_KEY` / `AWS_GOVCLOUD_BATCH_S3_BUCKET` / `AWS_GOVCLOUD_BATCH_ROLE_ARN`) | +| Bedrock split S3 identity | no | no | no | no | yes (file upload, content, delete) | S3 signed with `s3_access_key_id` / `s3_secret_access_key` (`AWS_S3_ONLY_ACCESS_KEY_ID` / `AWS_S3_ONLY_SECRET_ACCESS_KEY`, object rights on `AWS_BATCH_S3_BUCKET` only) while `aws_*` is `AWS_BEDROCK_ONLY_ACCESS_KEY_ID` / `AWS_BEDROCK_ONLY_SECRET_ACCESS_KEY`, an identity with no S3 rights on that bucket | Bedrock cancel maps to `StopModelInvocationJob` and comes back `cancelling`; the lifecycle asserts it the same way it does for OpenAI (`_CANCEL_ASSERTED_PROVIDERS`). diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index 9b3c06d9a1b..9bb6d05bec8 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -1024,6 +1024,72 @@ class TestBedrockBatchAssumeRole: assert fetched.id == batch.id +def _split_s3_identity_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=ASSUME_ROLE_RAW_MODEL, + aws_access_key_id="os.environ/AWS_BEDROCK_ONLY_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_BEDROCK_ONLY_SECRET_ACCESS_KEY", + aws_region_name="os.environ/AWS_REGION", + s3_region_name="os.environ/AWS_REGION", + s3_bucket_name="os.environ/AWS_BATCH_S3_BUCKET", + s3_access_key_id="os.environ/AWS_S3_ONLY_ACCESS_KEY_ID", + s3_secret_access_key="os.environ/AWS_S3_ONLY_SECRET_ACCESS_KEY", + aws_batch_role_arn="os.environ/AWS_BATCH_ROLE_ARN", + ) + + +class TestBedrockBatchSplitS3Credentials: + """Bedrock batch deployment whose aws_* identity cannot touch the bucket. + + AWS_BEDROCK_ONLY_* is an IAM user with no S3 rights on AWS_BATCH_S3_BUCKET; + AWS_S3_ONLY_* is an IAM user with object rights on that bucket only. Every + S3 call the proxy signs (PutObject on upload, GetObject on content, + DeleteObject on delete) must use the s3_* pair, otherwise S3 answers 403. + """ + + @pytest.mark.covers( + "llm.files.bedrock.split_s3_credentials.nonstream.works", + exercised_on=["files"], + ) + def test_file_lifecycle_signs_s3_with_s3_credentials( + self, client: BatchClient, resources: ResourceManager + ) -> None: + model_name = batch_model_name("bedrock-split-s3-batch") + model_id = client.create_model(model_name, _split_s3_identity_params()) + resources.defer(lambda: client.delete_model(model_id)) + key = resources.key() + + uploaded = client.upload_file( + content=render_jsonl(ASSUME_ROLE_RAW_MODEL), + form=FileUploadForm(purpose="batch", target_model_names=model_name), + key=key, + ) + assert isinstance(uploaded, Success), ( + f"upload must sign the S3 PutObject with s3_access_key_id, got {uploaded!r}" + ) + file = uploaded.data + resources.defer(lambda: cleanup_file(client, file.id, key=key)) + assert_file_object(file, provider="bedrock") + + downloaded = client.proxy.transport.download( + f"/v1/files/{file.id}/content", + headers=client.proxy.transport.bearer(key), + ) + assert downloaded.status_code == 200, ( + f"content must sign the S3 GetObject with s3_access_key_id, " + f"got {downloaded.status_code}: {downloaded.body[:300]}" + ) + assert all(json.loads(line) for line in downloaded.body.strip().splitlines()), ( + f"content download returned non-JSONL body: {downloaded.body[:200]}" + ) + + deleted = client.delete_file(file.id, key=key) + assert isinstance(deleted, Success), ( + f"delete must sign the S3 DeleteObject with s3_access_key_id, got {deleted!r}" + ) + assert deleted.data.id == file.id, f"delete confirmed a different file: {deleted.data!r}" + + GOVCLOUD_REGION: Final = "us-gov-west-1" GOVCLOUD_RAW_MODEL: Final = "bedrock/amazon.nova-lite-v1:0" diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 8776d00d502..f4de4ac01ba 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -30,6 +30,8 @@ from e2e_config import ( FIXTURE_MODE_RAW, MANAGED_FILES_OPT_IN_ENV, MCP_OAUTH_LIVE_OPT_IN_ENV, + OTEL_TLS_OPT_IN_ENV, + OTEL_V2_OPT_IN_ENV, PROMPT_CACHING_OPT_IN_ENV, PROVIDER_EDGE_HOST_OPT_IN_ENV, PROXY_BASE_URL, @@ -44,13 +46,18 @@ from fixture_mode import pytest_fixture_setup as pytest_fixture_setup from idp import Identity, Keycloak, keycloak_from_env from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager +from memory_readings import RssCapture, read_rss_everywhere from models import TeamNewBody, UserNewBody, UserNewResponse from provider_cache_routing import LIVE_PROVIDER_REQUIRED from provider_edge import replay_leftover_error from proxy_client import ProxyClient, build_proxy_client +from stack_lock import stack_lock _E2E_TEST_RAN = pytest.StashKey[bool]() _CALL_PASSED = pytest.StashKey[bool]() +_IDLE_RSS = pytest.StashKey[RssCapture]() + +IDLE_RSS_READ_TIMEOUT_SECONDS: Final = 10.0 OPT_IN_MARKERS: Final = MappingProxyType( { @@ -61,6 +68,8 @@ OPT_IN_MARKERS: Final = MappingProxyType( "cli_determinism": CLI_DETERMINISM_OPT_IN_ENV, "mcp_oauth_live": MCP_OAUTH_LIVE_OPT_IN_ENV, "provider_edge_host": PROVIDER_EDGE_HOST_OPT_IN_ENV, + "otel_v2": OTEL_V2_OPT_IN_ENV, + "otel_tls": OTEL_TLS_OPT_IN_ENV, } ) @@ -140,6 +149,11 @@ def pytest_configure(config: pytest.Config) -> None: "redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from " "gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set", ) + config.addinivalue_line( + "markers", + "quiet_stack: measures the proxy itself, so it runs while no other test on this host is hitting the stack; " + "every other test waits for it to finish", + ) config.addinivalue_line( "markers", "mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless " @@ -150,15 +164,21 @@ def pytest_configure(config: pytest.Config) -> None: "provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the " "gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set", ) + config.addinivalue_line( + "markers", + "otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set", + ) + config.addinivalue_line( + "markers", + "otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set", + ) def pytest_sessionstart(session: pytest.Session) -> None: """Abort before collection when E2E_FIXTURE_MODE can never work: an unknown mode value, or replay against a missing, unreadable, or stale bundle (the stale message names the bundle's age). Live and record modes pass through.""" - reason = fixture_mode_collection_error( - FIXTURE_MODE_RAW, FIXTURE_DIR, now=datetime.now(timezone.utc) - ) + reason = fixture_mode_collection_error(FIXTURE_MODE_RAW, FIXTURE_DIR, now=datetime.now(timezone.utc)) if reason is not None: raise pytest.UsageError(reason) @@ -174,6 +194,16 @@ def _needs_unset_opt_in(item: pytest.Item) -> bool: ) +def _reaches_proxy(item: pytest.Item) -> bool: + """True for a live test that talks to the shared proxy: `e2e`-marked and not a + `migration_startup` test, which boots its own container instead.""" + return item.get_closest_marker("e2e") is not None and item.get_closest_marker("migration_startup") is None + + +def _uses_idle_rss(item: pytest.Item) -> bool: + return isinstance(item, pytest.Function) and "idle_rss" in item.fixturenames + + def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: """Deselect every test behind an opt-in marker whose env var is unset (see OPT_IN_MARKERS): those tests need a proxy configured differently from the @@ -203,6 +233,20 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item items.sort(key=lambda item: item.get_closest_marker("load") is not None) +@pytest.hookimpl(tryfirst=True) +def pytest_collection_finish(session: pytest.Session) -> None: + """When a selected test asks for the `idle_rss` fixture and this is not a + `--collect-only` run, read every replica's RSS once, right here at the end of + collection and before this process sends any traffic. tryfirst keeps the read + ahead of xdist's own collection-finish report, and the controller schedules no + test until every worker has reported, so this is the idle footprint of a stack + that just passed its readiness gate. The fixture hands the capture to the + idle-budget test in router/test_reliability_memory_e2e.py.""" + if session.config.getoption("collectonly") or not any(_uses_idle_rss(item) for item in session.items): + return + session.config.stash[_IDLE_RSS] = read_rss_everywhere(build_proxy_client(), timeout=IDLE_RSS_READ_TIMEOUT_SECONDS) + + def _liveness_reason(label: str, base_url: str) -> str | None: """None if `base_url` answers its liveness probe, else a failure reason.""" try: @@ -227,6 +271,12 @@ def _proxy_fail_reason() -> str | None: return None +@pytest.hookimpl(wrapper=True) +def pytest_runtest_protocol(item: pytest.Item, nextitem: pytest.Item | None) -> Generator[None, object, object]: + with stack_lock(exclusive=item.get_closest_marker("quiet_stack") is not None): + return (yield) + + @pytest.hookimpl(tryfirst=True) def pytest_runtest_setup(item: pytest.Item) -> None: """Hard-fail `e2e`-marked tests unless a proxy answers its liveness probe. @@ -234,7 +284,9 @@ def pytest_runtest_setup(item: pytest.Item) -> None: run even when none is up. Never skip for a missing proxy. Replay mode needs the proxy too: only provider-bound traffic replays from the bundle.""" LIVE_PROVIDER_REQUIRED.set(item.get_closest_marker("provider_live") is not None) - if item.get_closest_marker("e2e") is None or item.get_closest_marker("migration_startup") is not None: + if _uses_idle_rss(item): + item.user_properties.extend(item.config.stash[_IDLE_RSS].junit_properties) + if not _reaches_proxy(item): return if isinstance(item, pytest.Function) and "oauth_gateway" in item.fixturenames: return @@ -249,7 +301,7 @@ def pytest_runtest_call(item: pytest.Item) -> None: guard before truncating the spend-log DB. Tests under `tests/e2e/` without the `e2e` marker (pure unit coverage for the harness itself) never hit the proxy, so they must not arm the destructive DB truncate.""" - if item.get_closest_marker("e2e") is None or item.get_closest_marker("migration_startup") is not None: + if not _reaches_proxy(item): return item.session.stash[_E2E_TEST_RAN] = True @@ -284,9 +336,7 @@ def pytest_runtest_teardown(item: pytest.Item) -> Generator[None, None, None]: LIVE_PROVIDER_REQUIRED.set(False) if not item.stash.get(_CALL_PASSED, False): return result - reason = replay_leftover_error( - mode_raw=FIXTURE_MODE_RAW, bundle_dir=FIXTURE_DIR, test_key=item.nodeid - ) + reason = replay_leftover_error(mode_raw=FIXTURE_MODE_RAW, bundle_dir=FIXTURE_DIR, test_key=item.nodeid) if reason is not None: pytest.fail(reason) return result @@ -313,6 +363,13 @@ def proxy() -> ProxyClient: return build_proxy_client() +@pytest.fixture(scope="session") +def idle_rss(request: pytest.FixtureRequest) -> RssCapture: + """Every replica's RSS as read once at the end of collection, before this process + sent any traffic (see pytest_collection_finish).""" + return request.config.stash[_IDLE_RSS] + + @pytest.fixture def resources(client: ProxyClientProvider) -> Iterator[ResourceManager]: """init -> run -> teardown: create a manager, run the test, release resources. diff --git a/tests/e2e/coverage_registry/guardrail.yaml b/tests/e2e/coverage_registry/guardrail.yaml index 86eb44f6cb1..f49568c883b 100644 --- a/tests/e2e/coverage_registry/guardrail.yaml +++ b/tests/e2e/coverage_registry/guardrail.yaml @@ -2,12 +2,15 @@ # Rolls up into the "Logging & Guardrails" dashboard module together with logging.* - {id: guardrail.presidio.pre_call.masks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "PII masking pre-call; data-leak blast radius"} - {id: guardrail.presidio.post_call.masks, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Mask PII in model output"} +- {id: guardrail.presidio.post_call.masks_generated_output, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, chat_completions_stream, anthropic_messages_stream], source: "guardrail_hooks/presidio.py", rationale: "Mask model-generated credit-card output with the UI default scope"} +- {id: guardrail.presidio.post_call.spend_log_stores_masked_output, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, chat_completions_stream, messages, anthropic_messages_stream, responses], source: "guardrail_hooks/presidio.py", fail_before_fix: proven, rationale: "When an output guardrail masks the response, the spend log stores the masked text the caller received rather than the raw model output, on every endpoint and both stream modes (LIT-8325)"} - {id: guardrail.presidio.logging_only.masks, module: guardrail, tier: P0, hook_point: logging_only, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Redact in logs without blocking"} - {id: guardrail.presidio.pre_call.logs_masked_entities, module: guardrail, tier: P0, hook_point: pre_call, assertions: [logs_masked_entities], exercised_on: [chat_completions], source: "guardrail_hooks/presidio.py", rationale: "A masking run must record itself on the spend log: the dashboard's guardrail panel renders the masked-entity counts and per-entity scores straight off metadata.guardrail_information, so a run that masks but records nothing leaves an operator unable to audit it"} - {id: guardrail.bedrock.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "AWS content guardrail blocks harmful input"} - {id: guardrail.litellm_content_filter.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "test_team_disable_global_guardrail_e2e.py", rationale: "Local content-filter default-on blocks banned keyword pre-call"} - {id: guardrail.litellm_content_filter.pre_call.blocks_video, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [videos], source: "test_key_guardrail_video_e2e.py", fail_before_fix: proven, rationale: "A content-filter guardrail attached to a key (metadata.guardrails) blocks a banned prompt on POST /v1/videos before the provider is called; before the fix the route's call type was unknown to the unified guardrail hook and the prompt went to the provider unscanned (LIT-6685)"} - {id: guardrail.litellm_content_filter.pre_call.allows, module: guardrail, tier: P0, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions], source: "test_team_disable_global_guardrail_e2e.py", rationale: "Team disable_global_guardrails bypasses default-on content filter"} +- {id: guardrail.litellm_content_filter.pre_call.returns_guardrail_information, module: guardrail, tier: P0, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions], source: "guardrails/test_guardrail_information_response_e2e.py", rationale: "Opt-in chat responses expose successful guardrail execution details"} - {id: guardrail.litellm_content_filter.apply_endpoint.blocks, module: guardrail, tier: P0, hook_point: apply_endpoint, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_endpoints.py:apply_guardrail", rationale: "POST /guardrails/apply_guardrail blocks banned content for customers that call the apply surface directly"} - {id: guardrail.litellm_content_filter.apply_endpoint.allows, module: guardrail, tier: P0, hook_point: apply_endpoint, assertions: [allows], exercised_on: [chat_completions], source: "guardrail_endpoints.py:apply_guardrail", rationale: "POST /guardrails/apply_guardrail returns clean text for allowed input"} - {id: guardrail.bedrock.during.blocks, module: guardrail, tier: P0, hook_point: during, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "During-call moderation for streaming"} diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 64335aa560c..e4e1ac2c7b6 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -89,11 +89,20 @@ - {id: llm.chat_completions.together_ai.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool result round trip"} - {id: llm.chat_completions.together_ai.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together cost header and spend row match the registry price"} - {id: llm.chat_completions.together_ai.thinking.nonstream.effort_none_disables, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: thinking, streaming: nonstream, assertions: [effort_none_disables], source: "llm_translation/test_together_ai_e2e.py", rationale: "reasoning_effort=none maps to Together's reasoning disable toggle on hybrid models"} -- {id: llm.chat_completions.xiaomi_mimo.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: xiaomi_mimo, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_xiaomi_mimo_e2e.py", rationale: "Native MiMo v2.6 rows price the cost header and spend row from the cost map"} -- {id: llm.chat_completions.xiaomi_mimo.thinking.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: xiaomi_mimo, capability: thinking, streaming: stream, assertions: [works], source: "llm_translation/test_xiaomi_mimo_e2e.py", rationale: "MiMo reasoning deltas stream as reasoning_content"} -- {id: llm.chat_completions.xiaomi_mimo.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: xiaomi_mimo, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_xiaomi_mimo_e2e.py", rationale: "MiMo tool calls are not dropped"} - {id: llm.chat_completions.together_ai.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: structured_output, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "response_format json_schema reaches Together and constrains the reply"} - {id: llm.chat_completions.together_ai.prompt_cache_5m.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: prompt_cache_5m, streaming: nonstream, assertions: [cache_hit, cost_logged], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together prefix-cache reads bill at cache_read_input_token_cost, not full input price"} - {id: llm.messages.together_ai.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together over /v1/messages streaming"} - {id: llm.messages.together_ai.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool calls over /v1/messages"} - {id: llm.messages.together_ai.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool result round trip over /v1/messages"} +- {id: llm.chat_completions.anthropic.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /chat/completions: cost header and spend row agree"} +- {id: llm.chat_completions.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /chat/completions"} +- {id: llm.messages.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /v1/messages"} +- {id: llm.messages.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI models served on the Anthropic Messages contract"} +- {id: llm.messages.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI over /v1/messages streams the Anthropic event grammar"} +- {id: llm.messages.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI over /v1/messages: cost header and spend row agree"} +- {id: llm.messages.openai.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI tool calls translated to Anthropic tool_use blocks"} +- {id: llm.messages.openai.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: openai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI tool result round trip over /v1/messages"} +- {id: llm.responses.openai.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "OpenAI function_call_output round trip over /v1/responses"} +- {id: llm.responses.anthropic.basic.stream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /v1/responses streams Responses events"} +- {id: llm.responses.anthropic.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /v1/responses: cost header and spend row agree"} +- {id: llm.responses.anthropic.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic function_call_output round trip over /v1/responses"} diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index 50f9b9808b2..c58c8af44ff 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -47,6 +47,7 @@ - {id: llm.files.vertex.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:52", rationale: "Vertex file upload to GCS"} - {id: llm.files.bedrock.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "batches/capabilities.py:59", rationale: "Bedrock file upload to S3"} - {id: llm.files.bedrock.govcloud_partition.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: bedrock_converse, capability: govcloud_partition, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Bedrock file upload to an S3 bucket in the us-gov-west-1 partition"} +- {id: llm.files.bedrock.split_s3_credentials.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: bedrock_converse, capability: split_s3_credentials, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-8297", rationale: "Bedrock file upload, content and delete sign S3 with s3_access_key_id / s3_secret_access_key when they differ from the aws_* identity"} - {id: llm.files.gemini.upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Gemini Files API upload via proxy"} - {id: llm.files.hosted_vllm.upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible file upload"} - {id: llm.files.openai.require_managed_files_upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "test_managed_files_enforcement_e2e.py / LIT-5902", rationale: "With require_managed_files enabled, an upload without target_model_names and an upload carrying a model param are both rejected 400; runs only in the sequential managed-files stack phase (E2E_MANAGED_FILES_STACK)"} diff --git a/tests/e2e/coverage_registry/logging.yaml b/tests/e2e/coverage_registry/logging.yaml index 1f2f1d64711..7c83e4d3aea 100644 --- a/tests/e2e/coverage_registry/logging.yaml +++ b/tests/e2e/coverage_registry/logging.yaml @@ -13,6 +13,7 @@ - {id: logging.otel.failure.exports_metric, module: logging, tier: P0, event: failure, assertions: [exports_metric], exercised_on: [chat_completions, messages], source: "integrations/otel/logger.py", rationale: "Error spans for observability continuity"} - {id: logging.braintrust.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions, messages], source: "integrations/braintrust_logging.py", rationale: "Evals platform spend"} - {id: logging.langsmith.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions, messages], source: "integrations/langsmith.py", rationale: "LangChain ecosystem"} +- {id: logging.langsmith.success.serializes_non_native_metadata, module: logging, tier: P1, event: success, assertions: [serializes_non_native_metadata], exercised_on: [sdk], source: "integrations/langsmith.py", rationale: "datetime/Decimal/UUID metadata used to TypeError in json.dumps and drop the whole batch (LIT-8310)"} - {id: logging.arize.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions, embeddings], source: "integrations/arize/arize.py", rationale: "ML-ops observability"} - {id: logging.mlflow.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions], source: "integrations/mlflow.py", rationale: "Experiment tracking cost/run"} - {id: logging.opik.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions], source: "integrations/opik/opik.py", rationale: "Eval platform spend/case"} diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index 85fbd0acd91..e1a840b1239 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -94,6 +94,10 @@ - {id: mgmt.mcp_toolset.update.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:3098", rationale: "Narrowing the tools to one entry reads back exactly that entry"} - {id: mgmt.mcp_toolset.update.clear_persists, module: mgmt, tier: P0, surface: api, assertions: [clear_persists], source: "mcp_management_endpoints.py:3098", fail_before_fix: proven, rationale: "An explicit null clears the stored description; the update used to drop null and keep the old value"} - {id: mgmt.mcp_toolset.delete.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:3149", rationale: "A deleted toolset is gone by id and from the list on every replica"} +- {id: mgmt.key.delete.audit_logged, module: mgmt, tier: P0, surface: api, assertions: [audit_logged], source: "key_management_endpoints.py:3981", rationale: "Every hard key deletion writes one LiteLLM_VerificationToken deleted audit row, whether the key is addressed by key or by alias"} +- {id: mgmt.team.member_delete.audit_logs_keys, module: mgmt, tier: P0, surface: api, assertions: [audit_logs_keys], source: "team_endpoints.py:3563", rationale: "Removing a team member hard-deletes their keys and each deleted key writes a deleted audit row"} +- {id: mgmt.team.delete.audit_logs_keys, module: mgmt, tier: P0, surface: api, assertions: [audit_logs_keys], source: "team_endpoints.py:4344", rationale: "Deleting a team hard-deletes its keys and each deleted key writes a deleted audit row"} +- {id: mgmt.user.delete.audit_logs_keys, module: mgmt, tier: P0, surface: api, assertions: [audit_logs_keys], source: "internal_user_endpoints.py:2369", rationale: "Deleting a user hard-deletes their keys and each deleted key writes a deleted audit row"} - {id: mgmt.user.jwt.database_roles, module: mgmt, tier: P0, surface: api, assertions: [database_roles], source: "auth/handle_jwt.py", rationale: "User-only JWT subjects retain their seeded database roles and memberships"} - {id: mgmt.key.jwt.viewer_denied, module: mgmt, tier: P0, surface: api, assertions: [viewer_denied], source: "auth/route_checks.py", rationale: "An admin viewer can read a key but cannot update it or change stored state"} diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index 9292e5f07db..1706b2f8f25 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -15,6 +15,8 @@ - {id: other.auth.jwt.invalid_signature_denied, module: other, tier: P0, area: auth, assertions: [invalid_signature_denied], source: "handle_jwt.py:1158-1166 _decode_jwt_with_public_key", rationale: "A genuine token whose signature bytes were altered fails verification with 401"} - {id: other.auth.jwt.unknown_team_denied, module: other, tier: P0, area: auth, assertions: [unknown_team_denied], source: "handle_jwt.py:1549-1632 find_team_with_model_access", rationale: "A verified JWT whose groups claim resolves to no existing team is denied with 403 naming the unresolved team, never silently admitted without a team. The proxy words it as a model-access denial, the same body an existing team without model access gets"} - {id: other.auth.jwt.virtual_key_unaffected, module: other, tier: P0, area: auth, assertions: [virtual_key_unaffected], source: "handle_jwt.py:213 is_jwt / user_api_key_auth.py:1332-1333", rationale: "enable_jwt_auth only routes three-segment bearer tokens into the JWT branch, so sk- virtual keys keep working on the same proxy"} +- {id: other.auth.jwt.team_header_alias_binds_team, module: other, tier: P0, area: auth, assertions: [team_header_alias_binds_team], source: "handle_jwt.py JWTAuthManager.resolve_team_from_header / LIT-7181", fail_before_fix: proven, rationale: "x-litellm-team-id carrying the team alias binds and attributes the same team as the team id, so a managed client can pin a stable alias instead of a uuid"} +- {id: other.auth.jwt.team_header_non_member_alias_denied, module: other, tier: P0, area: auth, assertions: [team_header_non_member_alias_denied], source: "handle_jwt.py JWTAuthManager.resolve_team_from_header / LIT-7181", rationale: "x-litellm-team-id naming the alias of a team the JWT does not grant is denied 403 with the same body as an unknown value, so the response does not reveal whether that team exists"} - {id: other.auth.model_access_group.wildcard_bare_name_allowed, module: other, tier: P0, area: auth, assertions: [wildcard_bare_name_allowed], source: "auth_checks.py:3232 / LIT-5813", fail_before_fix: proven, rationale: "A grant of a group holding a wildcard deployment covers the bare model names callers actually send, not only the provider-prefixed spelling"} - {id: other.auth.model_access_group.member_allowed, module: other, tier: P0, area: auth, assertions: [member_allowed], source: "auth_checks.py:3232", rationale: "A key whose allow-list is a model access group can call the deployments in that group"} - {id: other.auth.model_access_group.non_member_denied, module: other, tier: P0, area: auth, assertions: [non_member_denied], source: "auth_checks.py:3232", rationale: "That same grant reaches nothing outside the group, including provider models the group's wildcard does not cover"} diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index ad0914d455b..5ea48fdcd9d 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -10,6 +10,7 @@ - {id: quota_management.ratelimit.redis_backed.blocks_over_limit, module: quota_management, tier: P0, behavior: ratelimit, variant: redis_backed, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "parallel_request_limiter_v3.py", rationale: "With Redis configured, RPM still enforces 429 across the shared limiter path customers run multi-replica"} - {id: quota_management.ratelimit.rpm.resets_after_window, module: quota_management, tier: P1, behavior: ratelimit, variant: rpm, assertions: [resets_after_window], exercised_on: [chat_completions], source: "parallel_request_limiter_v3.py", rationale: "Rate-limit window (LITELLM_RATE_LIMIT_WINDOW_SIZE, 60s default) expires; a blocked key serves again in the next window"} - {id: quota_management.ratelimit.rpm.headers_report_remaining, module: quota_management, tier: P1, behavior: ratelimit, variant: rpm, assertions: [headers_report_remaining], exercised_on: [chat_completions], source: "parallel_request_limiter_v3.py async_post_call_success_hook", rationale: "Successful responses carry x-ratelimit-api_key-{limit,remaining}-{requests,tokens} so clients can pace"} +- {id: quota_management.ratelimit.model_group_alias.shares_bucket, module: quota_management, tier: P1, behavior: ratelimit, variant: model_group_alias, assertions: [shares_bucket], exercised_on: [chat_completions], source: "parallel_request_limiter_v3.py:_add_model_per_key_rate_limit_descriptor", rationale: "A model_group_alias draws on the same per-key deployment rpm bucket as the resolved model group, so alias plus real-name traffic cannot exceed the configured limit"} - {id: quota_management.ratelimit.priority_generous.picks_under_tpm, module: quota_management, tier: P1, behavior: ratelimit, variant: priority_generous, assertions: [picks_under_tpm], exercised_on: [chat_completions, messages], source: "dynamic_rate_limiter_v3.py:36-52", rationale: "Generous mode (<80% sat) allows priority borrowing"} - {id: quota_management.ratelimit.priority_strict.picks_under_tpm, module: quota_management, tier: P1, behavior: ratelimit, variant: priority_strict, assertions: [picks_under_tpm], exercised_on: [chat_completions, messages], source: "dynamic_rate_limiter_v3.py:53-71", rationale: "Strict mode (>=80% sat) enforces priority fairness"} - {id: quota_management.budget.key.blocks_over_limit, module: quota_management, tier: P0, behavior: budget, variant: key, assertions: [blocks_over_limit], exercised_on: [chat_completions], source: "proxy/auth/auth_checks.py", rationale: "A key's max_budget blocks further paid calls once spend crosses it"} @@ -45,10 +46,12 @@ - {id: quota_management.spend_tracking.cache_hit.zero_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_hit, assertions: [zero_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A response-cache hit logs at zero cost with the cache-hit marker"} - {id: quota_management.spend_tracking.key_rollup.matches_sum_of_logs, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_rollup, assertions: [matches_sum_of_logs], exercised_on: [chat_completions], source: "proxy/db/db_spend_update_writer.py", rationale: "A key's rolled-up spend equals the sum of its log rows"} - {id: quota_management.spend_tracking.concurrent_burst.loses_no_spend, module: quota_management, tier: P1, behavior: spend_tracking, variant: concurrent_burst, assertions: [loses_no_spend], exercised_on: [chat_completions], source: "proxy/db/db_spend_update_writer.py", rationale: "Concurrent calls all land as spend; no row lost to write contention"} +- {id: quota_management.spend_tracking.surface_consistency.matches_every_surface, module: quota_management, tier: P1, behavior: spend_tracking, variant: surface_consistency, assertions: [matches_every_surface], exercised_on: [chat_completions], source: "proxy/db/db_spend_update_writer.py", rationale: "One priced request lands the same response_cost on the spend log row, /key/info, /team/info, the usage export's /user/daily/activity/aggregated row, and the litellm_spend_metric Prometheus sample; each is a separate writer, so a rounding, dropped, or double-counted write on one drifts it from the rest (LIT-3620, LIT-5045)"} - {id: quota_management.spend_tracking.tags.attributes_spend, module: quota_management, tier: P1, behavior: spend_tracking, variant: tags, assertions: [attributes_spend], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Request tags round-trip to spend rows and tag rollups match tagged logs"} - {id: quota_management.spend_tracking.end_user.attributes_spend, module: quota_management, tier: P1, behavior: spend_tracking, variant: end_user, assertions: [attributes_spend], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "user= attribution lands the end-user id on the spend row"} - {id: quota_management.spend_tracking.per_model.writes_own_rows, module: quota_management, tier: P2, behavior: spend_tracking, variant: per_model, assertions: [writes_own_rows], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Each model on a shared key gets its own spend row"} - {id: quota_management.spend_tracking.failure.writes_failure_row, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_failure_row], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_log_error_logger.py", rationale: "A failed call writes a failure-status spend row"} +- {id: quota_management.spend_tracking.failure.attributes_provider, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [attributes_provider], exercised_on: [chat_completions], source: "proxy/utils.py", rationale: "A request rejected in pre_call_hook (rate limit, guardrail) still lands its single deployment's provider and model_id on the failure spend row"} - {id: quota_management.spend_tracking.spend_calculate.returns_cost, module: quota_management, tier: P2, behavior: spend_tracking, variant: spend_calculate, assertions: [returns_cost], exercised_on: [spend_calculate], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "/spend/calculate prices a hypothetical request at nonzero cost"} - {id: quota_management.spend_tracking.pagination.keeps_total, module: quota_management, tier: P2, behavior: spend_tracking, variant: pagination, assertions: [keeps_total], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "Spend-logs v2 pagination caps page size without losing the total"} - {id: quota_management.spend_tracking.cache_write.bills_cache_creation_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_write, assertions: [bills_cache_creation_rate], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "OpenAI cache-write tokens land on the spend row as cache-creation tokens billed at the cache-creation rate, not silently at the input rate (#34046)"} @@ -58,6 +61,7 @@ - {id: quota_management.spend_tracking.service_tier.bills_tier_rates, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier, assertions: [bills_tier_rates], exercised_on: [chat_completions], source: "cost_calculator.py", rationale: "A priority service_tier call bills input, output, and reasoning at the deployment's *_priority rates and records the tier on the row (#35923, #35925)"} - {id: quota_management.spend_tracking.cost_headers.additive_components, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_headers, assertions: [additive_components], exercised_on: [chat_completions], source: "proxy/common_request_processing.py", rationale: "The x-litellm-response-cost-* component headers sum to the total, input covers only fresh tokens, and reasoning stays a subset of output (#36965)"} - {id: quota_management.spend_tracking.passthrough_stream.injects_usage_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: passthrough_stream, assertions: [injects_usage_cost], exercised_on: [openai_passthrough], source: "proxy/pass_through_endpoints/streaming_handler.py", rationale: "With include_cost_in_streaming_usage on, the /openai passthrough's final streaming usage frame carries the proxy-computed cost (#36503). Uncovered: the flag is only settable in litellm_settings, and the shared e2e stack does not turn it on yet"} +- {id: quota_management.spend_tracking.websearch_interception.bills_under_request_session, module: quota_management, tier: P1, behavior: spend_tracking, variant: websearch_interception, assertions: [bills_under_request_session], exercised_on: [messages], source: "integrations/websearch_interception/handler.py", fail_before_fix: proven, rationale: "A web_search server tool the proxy intercepts into litellm.asearch writes its own asearch spend row, and that row carries the parent request's session_id so the session view counts the search and its cost next to the turn that triggered it (LIT-8063)"} - {id: quota_management.spend_tracking.key_attribution.joins_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [joins_key], exercised_on: [chat_completions, messages, responses, embeddings, batches, files, google_native, rust_control_plane], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Every spend row a virtual key writes across chat, queued chat, messages, responses, embeddings, the Gemini passthrough, file upload, batch create, and a replayed callback log carries api_key equal to the key's token hash and the key alias, the join the usage APIs depend on; a re-hashed token shows up as an unattributed key-hash-* row (#39568, #39572)"} - {id: quota_management.spend_tracking.key_attribution.reports_alias_and_email, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [reports_alias_and_email], exercised_on: [chat_completions, messages, responses, embeddings, batches, files, google_native, rust_control_plane], source: "proxy/management_endpoints/internal_user_endpoints.py", rationale: "/spend/logs?api_key= returns every one of the key's rows with its alias and /user/daily/activity aggregates them under the key's token with key_alias and user_email; /spend/logs carries no email field, so the email is asserted on daily activity only"} - {id: quota_management.spend_tracking.key_attribution.health_rows_keep_service_account, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [health_rows_keep_service_account], exercised_on: [chat_completions], source: "proxy/health_check.py", rationale: "A /health probe's spend row stays keyed by the literal litellm-internal-health-check service account rather than a hash of it, so health spend never appears as an unattributed key"} diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 437bd527927..367b6b8a454 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -9,6 +9,7 @@ - {id: reliability.retry.auth.succeeds_within_retries, module: reliability, tier: P1, behavior: retry, variant: auth, assertions: [succeeds_within_retries], exercised_on: [chat_completions], source: "get_retry_from_policy.py:42", rationale: "Transient auth glitch retry"} - {id: reliability.retry.context_window.succeeds_within_retries, module: reliability, tier: P1, behavior: retry, variant: context_window, assertions: [succeeds_within_retries], exercised_on: [chat_completions], source: "get_retry_from_policy.py:51", fail_before_fix: proven, rationale: "A context-window 400 under BadRequestErrorRetries retries onto a sibling deployment in the same model group, instead of coming straight back as the 400 the deployment that just refused it returned"} - {id: reliability.cooldown.5xx.trips_then_recovers, module: reliability, tier: P0, behavior: cooldown, variant: "5xx", assertions: [trips_then_recovers], exercised_on: [chat_completions], source: "cooldown_handlers.py:40", rationale: "Deployment cools after repeated 5xx, recovers after cooldown_time"} +- {id: reliability.cooldown.sibling_replica.serves_backup_within_read_interval, module: reliability, tier: P1, behavior: cooldown, variant: sibling_replica, assertions: [serves_backup_within_read_interval], exercised_on: [chat_completions], source: "cooldown_cache.py:44", fail_before_fix: proven, rationale: "A bench taken on one gateway reaches a sibling that already holds the key's read timer within the 1s Redis read interval plus margin, so its next call lands on the backup"} - {id: reliability.cooldown.429.trips_then_recovers, module: reliability, tier: P0, behavior: cooldown, variant: "429", assertions: [trips_then_recovers], exercised_on: [chat_completions], source: "cooldown_handlers.py:69", rationale: "Cools on 429, avoids hammering exhausted provider"} - {id: reliability.cooldown.auth.trips_then_recovers, module: reliability, tier: P1, behavior: cooldown, variant: auth, assertions: [trips_then_recovers], exercised_on: [chat_completions], source: "cooldown_handlers.py:74", rationale: "Cools on 401 auth error"} - {id: reliability.cooldown.timeout.trips_then_recovers, module: reliability, tier: P1, behavior: cooldown, variant: timeout, assertions: [trips_then_recovers], exercised_on: [chat_completions], source: "cooldown_handlers.py:77", rationale: "Cools on 408 timeout"} @@ -38,4 +39,5 @@ - {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"} - {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"} - {id: reliability.perf.memory.under_slo, module: reliability, tier: P1, behavior: perf, variant: memory, assertions: [under_slo], exercised_on: [chat_completions], source: grammar, rationale: "Proxy RSS and the stored request snapshot stay within fixed budgets across a few hundred failing requests with retries and fallbacks, the v1.100.0 retry-breadcrumb leak shape (MAT-335)"} +- {id: reliability.perf.idle_memory.under_slo, module: reliability, tier: P1, behavior: perf, variant: idle_memory, assertions: [under_slo], exercised_on: [], source: grammar, rationale: "Every worker's RSS as read right after the readiness gate and before any test traffic stays under a fixed idle budget; a DB-backed v1.100.x worker idled at 886 MB against a 2 GiB pod limit where v1.101.0rc1 idled at 544 MB"} - {id: reliability.perf.session_anomaly.under_slo, module: reliability, tier: P1, behavior: perf, variant: session_anomaly, assertions: [under_slo], exercised_on: [messages], source: grammar, rationale: "Weekly Claude Code-shaped multi-turn session load against real providers; ceilings on error rate, warm-turn cache read/write, p95 turn time, and gateway-recorded spend (LIT-4562)"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index d9c20d5c588..f3ac1ef8a83 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -67,6 +67,7 @@ LlmCapability = Literal[ "batch_deployment", "count_tokens", "govcloud_partition", + "split_s3_credentials", "input_validation", "long_context_1m", "mid_conversation_system", diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 14de4619664..ac26b2a3875 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -34,7 +34,7 @@ CONTROL_PLANE_BASE_URL = os.environ.get( def parse_replica_urls(raw: str, fallback: str) -> tuple[str, ...]: - urls: Final = tuple(url.strip().rstrip("/") for url in raw.split(",") if url.strip()) + urls: Final = tuple(dict.fromkeys(url.strip().rstrip("/") for url in raw.split(",") if url.strip())) return urls or (fallback,) @@ -58,6 +58,7 @@ LINEAR_READONLY_TOOL: Final = "list_teams" # as listed by tools/list on mcp.lin # service in docker-compose.yml maps it to host 16686). Trace-completeness tests # read exported spans back through it. OTEL_QUERY_URL = os.environ.get("E2E_OTEL_QUERY_URL", "http://localhost:16686").rstrip("/") +OTEL_EXPORTER_ENDPOINT = os.environ.get("E2E_OTEL_EXPORTER_ENDPOINT", "") # Real-DataDog read-back (no local sink - destination fakes cannot be deployed # on the cluster): the proxy delivers with DD_API_KEY as in production, and the @@ -147,6 +148,8 @@ REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS" CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM" MCP_OAUTH_LIVE_OPT_IN_ENV: Final = "E2E_MCP_OAUTH_LIVE" PROVIDER_EDGE_HOST_OPT_IN_ENV: Final = "E2E_PROVIDER_EDGE_HOST_REACHABLE" +OTEL_V2_OPT_IN_ENV: Final = "E2E_OTEL_V2" +OTEL_TLS_OPT_IN_ENV: Final = "E2E_OTEL_EXPORTER_ENDPOINT" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) @@ -170,6 +173,7 @@ MEMORY_CONCURRENCY = int(os.environ.get("E2E_MEMORY_CONCURRENCY", "4")) MEMORY_RSS_SETTLE_SAMPLES = int(os.environ.get("E2E_MEMORY_RSS_SETTLE_SAMPLES", "15")) MEMORY_RSS_SAMPLE_INTERVAL_SECONDS = float(os.environ.get("E2E_MEMORY_RSS_SAMPLE_INTERVAL_SECONDS", "1")) MEMORY_RSS_BUDGET_MB = float(os.environ.get("E2E_MEMORY_RSS_BUDGET_MB", "48")) +MEMORY_IDLE_RSS_BUDGET_MB = float(os.environ.get("E2E_MEMORY_IDLE_RSS_BUDGET_MB", "768")) MEMORY_STORED_REQUEST_BUDGET_KB = float(os.environ.get("E2E_MEMORY_STORED_REQUEST_BUDGET_KB", "64")) diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index 97f1e1671f8..1b62a1dbd8c 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -49,6 +49,7 @@ class AnthropicHeaders(AuthHeaders): on its own internal calls.""" anthropic_version: str = Field(default="2023-06-01", alias="anthropic-version") + x_litellm_session_id: str | None = Field(default=None, serialization_alias="x-litellm-session-id") class PartialBody(BaseModel): diff --git a/tests/e2e/gateway/record_replay_ci_config.yml b/tests/e2e/gateway/record_replay_ci_config.yml index 08972969cf0..bc1f3f10562 100644 --- a/tests/e2e/gateway/record_replay_ci_config.yml +++ b/tests/e2e/gateway/record_replay_ci_config.yml @@ -1,3 +1,7 @@ general_settings: master_key: os.environ/LITELLM_MASTER_KEY store_model_in_db: true + disable_model_info_refresh: true + +litellm_settings: + callbacks: ["prometheus"] diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml index 2d02fedddae..02a131f60ae 100644 --- a/tests/e2e/gateway/stage_mirror_ci_config.yml +++ b/tests/e2e/gateway/stage_mirror_ci_config.yml @@ -32,7 +32,10 @@ litellm_settings: - host: 127.0.0.1 port: 6379 ssl: true - callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel"] + callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel", "websearch_interception"] + websearch_interception_params: + enabled_providers: ["bedrock"] + search_tool_name: e2e-search require_auth_for_metrics_endpoint: false router_settings: @@ -40,6 +43,8 @@ router_settings: num_retries: 3 allowed_fails: 5 cooldown_time: 30 + model_group_alias: + e2e-alias-rl-alias: e2e-alias-rl-target model_list: - model_name: gpt-5.5 @@ -60,11 +65,22 @@ model_list: litellm_params: model: gemini/gemini-2.5-flash api_key: os.environ/GEMINI_API_KEY + - model_name: e2e-alias-rl-target + litellm_params: + model: anthropic/claude-haiku-4-5 + api_key: os.environ/ANTHROPIC_API_KEY + default_api_key_rpm_limit: 3 - model_name: openai-text-embedding-3-small litellm_params: model: openai/text-embedding-3-small api_key: os.environ/OPENAI_API_KEY +search_tools: + - search_tool_name: e2e-search + litellm_params: + search_provider: perplexity + api_key: os.environ/PERPLEXITY_API_KEY + files_settings: - custom_llm_provider: openai api_key: os.environ/OPENAI_API_KEY diff --git a/tests/e2e/guardrails/guardrails_client.py b/tests/e2e/guardrails/guardrails_client.py index 3ceac737399..60f875ccb7d 100644 --- a/tests/e2e/guardrails/guardrails_client.py +++ b/tests/e2e/guardrails/guardrails_client.py @@ -46,7 +46,7 @@ class BlockedWordBody(BaseModel): class GuardrailParamsBase(BaseModel): - mode: GuardrailMode + mode: GuardrailMode | list[GuardrailMode] default_on: bool @@ -291,6 +291,7 @@ class GuardrailsClient: text: str, *, guardrails: list[str] | None = None, + include_guardrail_response: bool | None = None, max_tokens: int = 16, tools: list[ChatTool] | None = None, ) -> Result[ChatResponse]: @@ -306,6 +307,7 @@ class GuardrailsClient: messages=[ChatMessage(role="user", content=text)], max_tokens=max_tokens, guardrails=guardrails, + include_guardrail_response=include_guardrail_response, tools=tools, ), ) @@ -381,6 +383,46 @@ class GuardrailsClient: ), ) + def messages_raw( + self, + key: str, + model: str, + text: str, + *, + guardrails: list[str] | None = None, + max_tokens: int = 64, + ) -> StreamingResponse: + return self.proxy.transport.send( + "/v1/messages", + headers=self.proxy.transport.bearer(key), + json=AnthropicMessagesBody( + model=model, + messages=[ChatMessage(role="user", content=text)], + max_tokens=max_tokens, + guardrails=guardrails, + ), + ) + + def messages_stream_raw( + self, + key: str, + model: str, + text: str, + *, + guardrails: list[str] | None = None, + max_tokens: int = 64, + ) -> StreamingResponse: + return self.proxy.messages_stream( + key, + AnthropicMessagesBody( + model=model, + messages=[ChatMessage(role="user", content=text)], + max_tokens=max_tokens, + stream=True, + guardrails=guardrails, + ), + ) + def responses( self, key: str, diff --git a/tests/e2e/guardrails/test_guardrail_information_response_e2e.py b/tests/e2e/guardrails/test_guardrail_information_response_e2e.py new file mode 100644 index 00000000000..9701ea35819 --- /dev/null +++ b/tests/e2e/guardrails/test_guardrail_information_response_e2e.py @@ -0,0 +1,118 @@ +"""Live e2e: an opted-in chat response includes the guardrail execution details.""" + +from __future__ import annotations + +import time +from typing import Final + +import pytest + +from e2e_config import unique_marker +from e2e_http import unwrap +from guardrails_client import ( + BlockedWordBody, + ContentFilterParamsBody, + GuardrailsClient, +) +from lifecycle import ResourceManager +from models import ChatResponse, GuardrailInformationEntry + +pytestmark = pytest.mark.e2e + +GUARDRAIL_PROPAGATION_DEADLINE_SECONDS: Final = 40.0 +GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS: Final = 5.0 + + +def _register_content_filter(client: GuardrailsClient, resources: ResourceManager, *, name: str) -> None: + guardrail_id = client.register( + name, + ContentFilterParamsBody( + mode="pre_call", + default_on=False, + blocked_words=[BlockedWordBody(keyword=f"never-match-{unique_marker()}", action="MASK")], + ), + ) + resources.defer(lambda: client.delete_guardrail(guardrail_id)) + + +def _opted_in_entries( + client: GuardrailsClient, + key: str, + model: str, + name: str, +) -> tuple[ChatResponse, tuple[GuardrailInformationEntry, ...]]: + response = unwrap( + client.chat( + key, + model, + "Reply with the single word OK.", + guardrails=[name], + include_guardrail_response=True, + max_tokens=16, + ) + ) + entries = tuple(entry for entry in response.guardrail_information or () if entry.guardrail_name == name) + return response, entries + + +class TestGuardrailInformationResponse: + @pytest.mark.covers( + "guardrail.litellm_content_filter.pre_call.returns_guardrail_information", + exercised_on=["chat_completions"], + ) + def test_flag_returns_guardrail_information_for_the_guardrail_that_ran( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"e2e-guardrail-information-{unique_marker()}" + _register_content_filter(client, resources, name=name) + model = client.create_backend_model( + resources, + prefix="e2e-guardrail-info-backend", + backend="openai/gpt-4.1-mini", + api_key="os.environ/OPENAI_API_KEY", + ) + deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS + + while True: + response, entries = _opted_in_entries(client, scoped_key, model, name) + if len(entries) == 1: + entry = entries[0] + assert entry.guardrail_status == "success", ( + f"guardrail information should report a successful run, got {entry!r}; response: {response}" + ) + assert entry.duration is not None and entry.duration >= 0, ( + f"guardrail information should report a non-negative duration; response: {response}" + ) + return + if time.monotonic() >= deadline: + pytest.fail( + f"guardrail information did not report exactly one successful {name!r} entry within " + f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; response: {response}" + ) + time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS) + + def test_without_flag_response_has_no_guardrail_information( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"e2e-guardrail-information-default-{unique_marker()}" + _register_content_filter(client, resources, name=name) + model = client.create_backend_model( + resources, + prefix="e2e-guardrail-info-backend", + backend="openai/gpt-4.1-mini", + api_key="os.environ/OPENAI_API_KEY", + ) + + response = unwrap( + client.chat( + scoped_key, + model, + "Reply with the single word OK.", + guardrails=[name], + max_tokens=16, + ) + ) + + assert "guardrail_information" not in response.model_fields_set, ( + f"guardrail information must remain absent without include_guardrail_response, got {response}" + ) diff --git a/tests/e2e/guardrails/test_presidio_masking_e2e.py b/tests/e2e/guardrails/test_presidio_masking_e2e.py index 49d698938ce..1fbc4a68bbd 100644 --- a/tests/e2e/guardrails/test_presidio_masking_e2e.py +++ b/tests/e2e/guardrails/test_presidio_masking_e2e.py @@ -29,7 +29,9 @@ this suite deliberately requires the detected-entity details to remain visible. from __future__ import annotations +import json import os +import re import time from collections.abc import Callable from typing import Final, Literal @@ -65,9 +67,13 @@ GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS = 5.0 # angle brackets; the logged payload keeps the placeholder verbatim. MASKED_EMAIL_TOKEN = "EMAIL_ADDRESS" MASKED_PHONE_TOKEN = "PHONE_NUMBER" +MASKED_CREDIT_CARD_TOKEN = "CREDIT_CARD" # Fictional NANP 555 number; a standard format Presidio's phone recognizer detects. FAKE_PHONE = "+1 415-555-0134" +FAKE_VISA_TEST_CARD = "4111 1111 1111 1111" + +_CARD_DIGIT_RUN: Final = re.compile(r"(?:\d[ -]?){13,19}") def _presidio_bases() -> tuple[str, str]: @@ -86,8 +92,8 @@ def _register_presidio( resources: ResourceManager, *, name: str, - mode: GuardrailMode = "pre_call", - filter_scope: Literal["input", "output", "both"] = "input", + mode: GuardrailMode | list[GuardrailMode] = "pre_call", + filter_scope: Literal["input", "output", "both"] | None = "input", entities: dict[PiiEntity, PiiAction] | None = None, ) -> None: analyzer, anonymizer = _presidio_bases() @@ -123,6 +129,74 @@ def _first_content(response: ChatResponse) -> str: return (message.content if message else None) or "" +class _StreamDelta(BaseModel): + content: str | None = None + + +class _StreamChoice(BaseModel): + delta: _StreamDelta + + +class _StreamChunk(BaseModel): + choices: tuple[_StreamChoice, ...] = () + + +class _AnthropicStreamDelta(BaseModel): + type: str | None = None + text: str | None = None + + +class _AnthropicStreamEvent(BaseModel): + type: str + delta: _AnthropicStreamDelta | None = None + + +def _credit_card_prompt(marker: str) -> str: + return ( + f"{marker} Reply with only the well known Visa sandbox test card number that starts with 4111, " + "the 16 digits grouped in fours separated by spaces, and nothing else." + ) + + +def _passes_luhn(digits: str) -> bool: + checksum = sum( + digit if position % 2 == 0 else (digit * 2 - 9 if digit * 2 > 9 else digit * 2) + for position, digit in enumerate(int(char) for char in reversed(digits)) + ) + return checksum % 10 == 0 + + +def _contains_card_number(text: str) -> bool: + """Presidio's CREDIT_CARD recognizer only reports Luhn-valid digit runs, so a + Luhn-invalid number the model hallucinates is not something masking can catch.""" + return any( + 13 <= len(digits) <= 19 and _passes_luhn(digits) + for digits in (re.sub(r"[ -]", "", match.group()) for match in _CARD_DIGIT_RUN.finditer(text)) + ) + + +def _stream_content(result: StreamingResponse) -> str: + return "".join( + choice.delta.content + for event in result.stream_events + if event != "[DONE]" + for choice in _StreamChunk.model_validate_json(event).choices[:1] + if choice.delta.content + ) + + +def _anthropic_stream_content(result: StreamingResponse) -> str: + return "".join( + event.delta.text + for payload in result.stream_events + for event in [_AnthropicStreamEvent.model_validate_json(payload)] + if event.type == "content_block_delta" + and event.delta is not None + and event.delta.type == "text_delta" + and event.delta.text + ) + + def _messages_text(response: AnthropicMessagesResponse) -> str: """The text of a /v1/messages answer, whichever shape the proxy produced (Anthropic-native content blocks or OpenAI-normalized choices).""" @@ -290,6 +364,267 @@ class TestPresidioPostCallMasking: time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS) +def _assert_eventually_masks_generated_card(fetch: Callable[[], str | None]) -> None: + deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS + last: str = "" + while True: + content = fetch() + if content is not None: + last = content + if _contains_card_number(content): + pytest.fail( + "the post_call output masking let a card number through: " + f"{content[:300]!r}" + ) + if MASKED_CREDIT_CARD_TOKEN in content: + return + if time.monotonic() >= deadline: + pytest.fail( + "presidio post_call output masking never masked the generated card within " + f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; last observation: {last[:300]!r}" + ) + time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS) + + +class TestPresidioCreditCardOutputMasking: + """Proves the UI-default Presidio scope masks model-generated card output.""" + + @pytest.mark.covers( + "guardrail.presidio.post_call.masks_generated_output", + exercised_on=["chat_completions"], + ) + def test_ui_default_scope_masks_a_card_number_the_model_generates_on_chat_completions( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"e2e-presidio-card-chat-{unique_marker()}" + _register_presidio( + client, + resources, + name=name, + mode=["pre_call", "post_call"], + filter_scope=None, + entities={"CREDIT_CARD": "MASK"}, + ) + prompt: Final = _credit_card_prompt(unique_marker()) + + def fetch() -> str | None: + result: Final = client.chat(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=512) + match result: + case Success(data=data): + return _first_content(data) + case _: + return None + + _assert_eventually_masks_generated_card(fetch) + + @pytest.mark.covers( + "guardrail.presidio.post_call.masks_generated_output", + exercised_on=["chat_completions_stream"], + ) + def test_ui_default_scope_masks_a_card_number_the_model_generates_on_streaming_chat_completions( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"e2e-presidio-card-stream-{unique_marker()}" + _register_presidio( + client, + resources, + name=name, + mode=["pre_call", "post_call"], + filter_scope=None, + entities={"CREDIT_CARD": "MASK"}, + ) + prompt: Final = _credit_card_prompt(unique_marker()) + + def fetch() -> str | None: + result: Final = client.chat_stream_raw( + scoped_key, + MODEL, + prompt, + guardrails=[name], + max_tokens=512, + ) + if not result.ok or result.stream_error: + return None + return _stream_content(result) + + _assert_eventually_masks_generated_card(fetch) + + @pytest.mark.covers( + "guardrail.presidio.post_call.masks_generated_output", + exercised_on=["anthropic_messages_stream"], + ) + def test_ui_default_scope_masks_a_card_number_the_model_generates_on_streaming_anthropic_messages( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + name = f"e2e-presidio-card-messages-stream-{unique_marker()}" + _register_presidio( + client, + resources, + name=name, + mode=["pre_call", "post_call"], + filter_scope=None, + entities={"CREDIT_CARD": "MASK"}, + ) + prompt: Final = _credit_card_prompt(unique_marker()) + + def fetch() -> str | None: + result: Final = client.messages_stream_raw( + scoped_key, + MODEL, + prompt, + guardrails=[name], + max_tokens=512, + ) + if not result.ok or result.stream_error: + return None + return _anthropic_stream_content(result) + + _assert_eventually_masks_generated_card(fetch) + + +def _wire_text(outcome: StreamingResponse) -> str: + return "\n".join(outcome.stream_events) if outcome.is_streaming else outcome.body + + +def _poll_until_generated_card_masked(fetch: Callable[[], StreamingResponse]) -> StreamingResponse: + """The raw HTTP outcome once the output masker is in effect on the serving + worker: whichever wire shape the endpoint speaks, a masked body carries the + CREDIT_CARD placeholder and no Luhn-valid card run. A raw card is a worker + that has not loaded the guardrail yet, so it is polled through like any + other unmasked answer.""" + deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS + last: str = "" + while True: + outcome = fetch() + if outcome.ok and not outcome.stream_error: + wire = _wire_text(outcome) + last = wire + if MASKED_CREDIT_CARD_TOKEN in wire and not _contains_card_number(wire): + return outcome + if time.monotonic() >= deadline: + pytest.fail( + "presidio post_call output masking never masked the generated card within " + f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; last observation: {last[:300]!r}" + ) + time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS) + + +def _spend_log_response_text(client: GuardrailsClient, key: str, call_id: str) -> str: + rows = client.proxy.poll_logs_for_key( + key, + predicate=lambda logged: any(row.litellm_call_id == call_id for row in logged), + ) + row = next((row for row in rows if row.litellm_call_id == call_id), None) + assert row is not None, f"no spend log row ever appeared for x-litellm-call-id {call_id}" + return json.dumps(row.response) + + +class TestPresidioSpendLogStoresMaskedOutput: + """The spend log stores the response the caller received, on every endpoint + and both stream modes, when an output-only post_call Presidio guardrail masks + a card number the model generated.""" + + _CELL: Final = "guardrail.presidio.post_call.spend_log_stores_masked_output" + + def _assert_spend_log_is_masked( + self, + client: GuardrailsClient, + resources: ResourceManager, + key: str, + *, + name: str, + fetch: Callable[[str, str], StreamingResponse], + ) -> None: + _register_presidio( + client, + resources, + name=name, + mode="post_call", + filter_scope="output", + entities={"CREDIT_CARD": "MASK"}, + ) + prompt: Final = _credit_card_prompt(unique_marker()) + outcome = _poll_until_generated_card_masked(lambda: fetch(prompt, name)) + assert outcome.call_id, f"the served response must carry x-litellm-call-id: {dict(outcome.headers)}" + + logged = _spend_log_response_text(client, key, outcome.call_id) + assert not _contains_card_number(logged), ( + "the caller got the masked response but the spend log stored the raw model output: " + f"{logged[:400]!r}" + ) + assert MASKED_CREDIT_CARD_TOKEN in logged, ( + f"the spend log response carries neither the card nor the placeholder: {logged[:400]!r}" + ) + + @pytest.mark.covers(_CELL, exercised_on=["chat_completions"]) + def test_spend_log_stores_masked_output_on_chat_completions( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + self._assert_spend_log_is_masked( + client, + resources, + scoped_key, + name=f"e2e-presidio-log-card-chat-{unique_marker()}", + fetch=lambda prompt, guardrail: client.chat_raw( + scoped_key, MODEL, prompt, guardrails=[guardrail], max_tokens=512 + ), + ) + + @pytest.mark.covers(_CELL, exercised_on=["chat_completions_stream"]) + def test_spend_log_stores_masked_output_on_streaming_chat_completions( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + self._assert_spend_log_is_masked( + client, + resources, + scoped_key, + name=f"e2e-presidio-log-card-chat-stream-{unique_marker()}", + fetch=lambda prompt, guardrail: client.chat_stream_raw( + scoped_key, MODEL, prompt, guardrails=[guardrail], max_tokens=512 + ), + ) + + @pytest.mark.covers(_CELL, exercised_on=["messages"]) + def test_spend_log_stores_masked_output_on_anthropic_messages( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + self._assert_spend_log_is_masked( + client, + resources, + scoped_key, + name=f"e2e-presidio-log-card-messages-{unique_marker()}", + fetch=lambda prompt, guardrail: client.messages_raw( + scoped_key, MODEL, prompt, guardrails=[guardrail], max_tokens=512 + ), + ) + + @pytest.mark.covers(_CELL, exercised_on=["anthropic_messages_stream"]) + def test_spend_log_stores_masked_output_on_streaming_anthropic_messages( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + self._assert_spend_log_is_masked( + client, + resources, + scoped_key, + name=f"e2e-presidio-log-card-messages-stream-{unique_marker()}", + fetch=lambda prompt, guardrail: client.messages_stream_raw( + scoped_key, MODEL, prompt, guardrails=[guardrail], max_tokens=512 + ), + ) + + @pytest.mark.covers(_CELL, exercised_on=["responses"]) + def test_spend_log_stores_masked_output_on_responses( + self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str + ) -> None: + self._assert_spend_log_is_masked( + client, + resources, + scoped_key, + name=f"e2e-presidio-log-card-responses-{unique_marker()}", + fetch=lambda prompt, guardrail: client.responses(scoped_key, MODEL, prompt, guardrails=[guardrail]), + ) + + _LOGGED_ENTITIES: dict[PiiEntity, PiiAction] = {"EMAIL_ADDRESS": "MASK", "PHONE_NUMBER": "MASK"} _ENTITY_LIST_ADAPTER: Final = TypeAdapter(list[GuardrailEntityMatch]) diff --git a/tests/e2e/llm_translation/conversational_matrix.py b/tests/e2e/llm_translation/conversational_matrix.py new file mode 100644 index 00000000000..0d6f6ed3d4e --- /dev/null +++ b/tests/e2e/llm_translation/conversational_matrix.py @@ -0,0 +1,544 @@ +"""The endpoint x deployment x auth matrix behind test_conversational_matrix_e2e.py. + +One conversation, three wire formats. Each `Surface` speaks its own API through +the customer SDK (chat completions and Responses through the OpenAI SDK, Messages +through the Anthropic SDK) and folds what came back into the surface-neutral +`Reply` / `StreamedReply`, so a single behavior test asserts the same contract on +every cell. A new model, from an existing or a new provider, is one `Deployment` row +in DEPLOYMENTS; a new way of handing the proxy a provider credential is one +`AuthMethod`. +""" + +from __future__ import annotations + +import json +import os +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, Literal, Protocol + +import anthropic +import openai +import pytest +from _pytest.mark.structures import ParameterSet +from anthropic.types import ( + MessageParam, + RawMessageStreamEvent, + TextBlock, + ToolChoiceToolParam, + ToolParam, + ToolResultBlockParam, + ToolUseBlock, + ToolUseBlockParam, +) +from e2e_config import provider_edge_base, unique_marker +from lifecycle import ResourceManager +from llm_translation.sdk_clients import NO_PROXY_CACHE, SdkClients, response_header +from models import CredentialCreateBody, LiteLLMParamsBody +from openai.types.chat import ( + ChatCompletionAssistantMessageParam, + ChatCompletionChunk, + ChatCompletionMessageFunctionToolCallParam, + ChatCompletionMessageParam, + ChatCompletionNamedToolChoiceParam, + ChatCompletionToolMessageParam, + ChatCompletionToolParam, +) +from openai.types.chat.chat_completion_message_function_tool_call import ChatCompletionMessageFunctionToolCall +from openai.types.responses import ( + FunctionToolParam, + ResponseFunctionToolCall, + ResponseFunctionToolCallParam, + ResponseInputParam, + ResponseStreamEvent, + ToolChoiceFunctionParam, +) +from openai.types.responses.response_input_param import FunctionCallOutput +from proxy_client import ProxyClient +from pydantic import BaseModel + +SurfaceName = Literal["chat_completions", "messages", "responses"] +AuthMethod = Literal["env_ref", "stored_credential"] +Capability = Literal["basic", "tool_use", "multi_turn"] +Streaming = Literal["stream", "nonstream"] +Assertion = Literal["works", "cost_logged"] +ToolMode = Literal["none", "forced", "offered"] + +SURFACES: Final[tuple[SurfaceName, ...]] = ("chat_completions", "messages", "responses") +AUTH_METHODS: Final[tuple[AuthMethod, ...]] = ("env_ref", "stored_credential") + +MAX_OUTPUT_TOKENS: Final = 512 +INSTRUCTIONS: Final = "You are a terse assistant. Answer in one short sentence." +GREETING_PROMPT: Final = "Say hello." +WEATHER_PROMPT: Final = "What is the weather in Paris right now? Use the get_weather tool." +WEATHER_REPORT: Final = "Paris: 22 degrees Celsius, clear skies" +WEATHER_TOOL_NAME: Final = "get_weather" +WEATHER_TOOL_DESCRIPTION: Final = "Current weather for a city" +WEATHER_TOOL_SCHEMA: Final[Mapping[str, object]] = MappingProxyType( + { + "type": "object", + "properties": {"location": {"type": "string", "description": "City name"}}, + "required": ["location"], + } +) + + +@dataclass(frozen=True, slots=True) +class Deployment: + """One deployment target: the litellm backend string plus how to wire it.""" + + route: Literal["openai", "anthropic"] + label: str + backend: str + api_key_env: str + edge_mount: str + edge_suffix: str + + def api_base(self) -> str | None: + base: Final = provider_edge_base(self.edge_mount) + return None if base is None else f"{base}{self.edge_suffix}" + + def api_key(self) -> str: + key: Final = os.environ.get(self.api_key_env, "") + assert key, f"{self.api_key_env} is not set in the test process environment" + return key + + +DEPLOYMENTS: Final[tuple[Deployment, ...]] = ( + Deployment( + route="openai", + label="gpt-4o-mini", + backend="openai/gpt-4o-mini", + api_key_env="OPENAI_API_KEY", + edge_mount="openai", + edge_suffix="/v1", + ), + Deployment( + route="openai", + label="gpt-5.4-mini", + backend="openai/gpt-5.4-mini", + api_key_env="OPENAI_API_KEY", + edge_mount="openai", + edge_suffix="/v1", + ), + Deployment( + route="anthropic", + label="claude-haiku-4-5", + backend="anthropic/claude-haiku-4-5", + api_key_env="ANTHROPIC_API_KEY", + edge_mount="anthropic", + edge_suffix="", + ), +) + + +@dataclass(frozen=True, slots=True) +class Cell: + surface: SurfaceName + deployment: Deployment + auth: AuthMethod + + @property + def id(self) -> str: + return f"{self.surface}-{self.deployment.label}-{self.auth}" + + def registry_id(self, capability: Capability, streaming: Streaming, assertion: Assertion) -> str: + return f"llm.{self.surface}.{self.deployment.route}.{capability}.{streaming}.{assertion}" + + +CELLS: Final[tuple[Cell, ...]] = tuple( + Cell(surface=surface, deployment=deployment, auth=auth) + for surface in SURFACES + for deployment in DEPLOYMENTS + for auth in AUTH_METHODS +) + + +def cells_covering(capability: Capability, streaming: Streaming, assertion: Assertion) -> tuple[ParameterSet, ...]: + """Every cell as a pytest param carrying the registry id its test proves.""" + return tuple( + pytest.param(cell, id=cell.id, marks=pytest.mark.covers(cell.registry_id(capability, streaming, assertion))) + for cell in CELLS + ) + + +DeploymentKey = tuple[str, AuthMethod] + + +@dataclass(frozen=True, slots=True) +class Deployments: + """Model aliases registered on the proxy, one per (deployment, auth).""" + + aliases: Mapping[DeploymentKey, str] + + def alias(self, cell: Cell) -> str: + return self.aliases[(cell.deployment.label, cell.auth)] + + +def _litellm_params(deployment: Deployment, auth: AuthMethod, credential_name: str) -> LiteLLMParamsBody: + match auth: + case "env_ref": + return LiteLLMParamsBody( + model=deployment.backend, api_key=f"os.environ/{deployment.api_key_env}", api_base=deployment.api_base() + ) + case "stored_credential": + return LiteLLMParamsBody( + model=deployment.backend, litellm_credential_name=credential_name, api_base=deployment.api_base() + ) + + +def _register(proxy: ProxyClient, resources: ResourceManager, deployment: Deployment, auth: AuthMethod) -> str: + marker: Final = unique_marker() + credential_name: Final = f"e2e-matrix-{deployment.label}-{marker}" + if auth == "stored_credential": + proxy.create_credential( + CredentialCreateBody(credential_name=credential_name, credential_values={"api_key": deployment.api_key()}) + ) + resources.defer(lambda: proxy.delete_credential(credential_name)) + alias: Final = f"e2e-matrix-{deployment.label}-{auth}-{marker}" + model_id: Final = proxy.create_model(alias, _litellm_params(deployment, auth, credential_name)) + resources.defer(lambda: proxy.delete_model(model_id)) + return alias + + +def register_deployments(proxy: ProxyClient) -> Iterator[Deployments]: + resources: Final = ResourceManager(client=proxy) + try: + yield Deployments( + aliases=MappingProxyType( + { + (deployment.label, auth): _register(proxy, resources, deployment, auth) + for deployment in DEPLOYMENTS + for auth in AUTH_METHODS + } + ) + ) + finally: + resources.teardown() + + +class WeatherArgs(BaseModel): + location: str + + +@dataclass(frozen=True, slots=True) +class ToolCall: + call_id: str + name: str + arguments: str + + def parsed(self) -> WeatherArgs: + return WeatherArgs.model_validate_json(self.arguments) + + +@dataclass(frozen=True, slots=True) +class Usage: + input_tokens: int + output_tokens: int + + +@dataclass(frozen=True, slots=True) +class Reply: + """What every surface owes the caller for one non-streamed turn.""" + + response_id: str + text: str + tool_calls: tuple[ToolCall, ...] + usage: Usage | None + call_id_header: str | None + cost_header: str | None + + +@dataclass(frozen=True, slots=True) +class StreamedReply: + """The reassembled stream: its text, whether the surface's own terminal event + arrived, and whether usage was reported anywhere in the stream.""" + + text: str + finished: bool + usage_reported: bool + event_count: int + + +class Surface(Protocol): + @property + def name(self) -> SurfaceName: ... + + def reply(self, key: str, model: str, prompt: str, *, with_tool: bool = False) -> Reply: ... + + def stream(self, key: str, model: str, prompt: str) -> StreamedReply: ... + + def reply_to_tool_result(self, key: str, model: str, prompt: str, call: ToolCall, result: str) -> Reply: ... + + +def _chat_tool() -> ChatCompletionToolParam: + return { + "type": "function", + "function": { + "name": WEATHER_TOOL_NAME, + "description": WEATHER_TOOL_DESCRIPTION, + "parameters": dict(WEATHER_TOOL_SCHEMA), + }, + } + + +def _messages_tool() -> ToolParam: + return { + "name": WEATHER_TOOL_NAME, + "description": WEATHER_TOOL_DESCRIPTION, + "input_schema": dict(WEATHER_TOOL_SCHEMA), + } + + +def _responses_tool() -> FunctionToolParam: + return { + "type": "function", + "name": WEATHER_TOOL_NAME, + "description": WEATHER_TOOL_DESCRIPTION, + "parameters": dict(WEATHER_TOOL_SCHEMA), + "strict": False, + } + + +def _chat_tool_choice() -> ChatCompletionNamedToolChoiceParam: + return {"type": "function", "function": {"name": WEATHER_TOOL_NAME}} + + +def _messages_tool_choice() -> ToolChoiceToolParam: + return {"type": "tool", "name": WEATHER_TOOL_NAME, "disable_parallel_tool_use": True} + + +def _responses_tool_choice() -> ToolChoiceFunctionParam: + return {"type": "function", "name": WEATHER_TOOL_NAME} + + +def _usage(input_tokens: int | None, output_tokens: int | None) -> Usage | None: + if input_tokens is None or output_tokens is None: + return None + return Usage(input_tokens=input_tokens, output_tokens=output_tokens) + + +@dataclass(frozen=True, slots=True) +class ChatCompletionsSurface: + sdk: SdkClients + name: SurfaceName = "chat_completions" + + def _turn(self, key: str, model: str, messages: Sequence[ChatCompletionMessageParam], tool: ToolMode) -> Reply: + raw: Final = self.sdk.openai(key).chat.completions.with_raw_response.create( + model=model, + messages=list(messages), + max_completion_tokens=MAX_OUTPUT_TOKENS, + tools=openai.omit if tool == "none" else [_chat_tool()], + tool_choice=_chat_tool_choice() if tool == "forced" else openai.omit, + parallel_tool_calls=False if tool == "forced" else openai.omit, + extra_body=NO_PROXY_CACHE, + ) + completion: Final = raw.parse() + message: Final = completion.choices[0].message + calls: Final = tuple( + ToolCall(call_id=call.id, name=call.function.name, arguments=call.function.arguments) + for call in message.tool_calls or () + if isinstance(call, ChatCompletionMessageFunctionToolCall) + ) + return Reply( + response_id=completion.id, + text=message.content or "", + tool_calls=calls, + usage=None + if completion.usage is None + else _usage(completion.usage.prompt_tokens, completion.usage.completion_tokens), + call_id_header=response_header(raw.headers, "x-litellm-call-id"), + cost_header=response_header(raw.headers, "x-litellm-response-cost"), + ) + + def reply(self, key: str, model: str, prompt: str, *, with_tool: bool = False) -> Reply: + return self._turn(key, model, _chat_history(prompt), "forced" if with_tool else "none") + + def stream(self, key: str, model: str, prompt: str) -> StreamedReply: + chunks: Final[tuple[ChatCompletionChunk, ...]] = tuple( + self.sdk.openai(key).chat.completions.create( + model=model, + messages=_chat_history(prompt), + max_completion_tokens=MAX_OUTPUT_TOKENS, + stream=True, + stream_options={"include_usage": True}, + extra_body=NO_PROXY_CACHE, + ) + ) + return StreamedReply( + text="".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices), + finished=any(chunk.choices[0].finish_reason is not None for chunk in chunks if chunk.choices), + usage_reported=any(chunk.usage is not None for chunk in chunks), + event_count=len(chunks), + ) + + def reply_to_tool_result(self, key: str, model: str, prompt: str, call: ToolCall, result: str) -> Reply: + tool_call: Final[ChatCompletionMessageFunctionToolCallParam] = { + "id": call.call_id, + "type": "function", + "function": {"name": call.name, "arguments": call.arguments}, + } + assistant: Final[ChatCompletionAssistantMessageParam] = {"role": "assistant", "tool_calls": [tool_call]} + tool_result: Final[ChatCompletionToolMessageParam] = { + "role": "tool", + "tool_call_id": call.call_id, + "content": result, + } + return self._turn(key, model, (*_chat_history(prompt), assistant, tool_result), "offered") + + +def _chat_history(prompt: str) -> tuple[ChatCompletionMessageParam, ...]: + return ({"role": "system", "content": INSTRUCTIONS}, {"role": "user", "content": prompt}) + + +@dataclass(frozen=True, slots=True) +class MessagesSurface: + sdk: SdkClients + name: SurfaceName = "messages" + + def _turn(self, key: str, model: str, messages: Sequence[MessageParam], tool: ToolMode) -> Reply: + raw: Final = self.sdk.anthropic(key).messages.with_raw_response.create( + model=model, + max_tokens=MAX_OUTPUT_TOKENS, + system=INSTRUCTIONS, + messages=list(messages), + tools=anthropic.omit if tool == "none" else [_messages_tool()], + tool_choice=_messages_tool_choice() if tool == "forced" else anthropic.omit, + extra_body=NO_PROXY_CACHE, + ) + message: Final = raw.parse() + return Reply( + response_id=message.id, + text="".join(block.text for block in message.content if isinstance(block, TextBlock)), + tool_calls=tuple( + ToolCall(call_id=block.id, name=block.name, arguments=json.dumps(block.input)) + for block in message.content + if isinstance(block, ToolUseBlock) + ), + usage=_usage(message.usage.input_tokens, message.usage.output_tokens), + call_id_header=response_header(raw.headers, "x-litellm-call-id"), + cost_header=response_header(raw.headers, "x-litellm-response-cost"), + ) + + def reply(self, key: str, model: str, prompt: str, *, with_tool: bool = False) -> Reply: + return self._turn(key, model, ({"role": "user", "content": prompt},), "forced" if with_tool else "none") + + def stream(self, key: str, model: str, prompt: str) -> StreamedReply: + events: Final[tuple[RawMessageStreamEvent, ...]] = tuple( + self.sdk.anthropic(key).messages.create( + model=model, + max_tokens=MAX_OUTPUT_TOKENS, + system=INSTRUCTIONS, + messages=[{"role": "user", "content": prompt}], + stream=True, + extra_body=NO_PROXY_CACHE, + ) + ) + return StreamedReply( + text="".join( + event.delta.text + for event in events + if event.type == "content_block_delta" and event.delta.type == "text_delta" + ), + finished=any(event.type == "message_stop" for event in events), + usage_reported=any(event.type == "message_delta" and event.usage.output_tokens > 0 for event in events), + event_count=len(events), + ) + + def reply_to_tool_result(self, key: str, model: str, prompt: str, call: ToolCall, result: str) -> Reply: + tool_use: Final[ToolUseBlockParam] = { + "type": "tool_use", + "id": call.call_id, + "name": call.name, + "input": call.parsed().model_dump(), + } + tool_result: Final[ToolResultBlockParam] = { + "type": "tool_result", + "tool_use_id": call.call_id, + "content": result, + } + history: Final[tuple[MessageParam, ...]] = ( + {"role": "user", "content": prompt}, + {"role": "assistant", "content": [tool_use]}, + {"role": "user", "content": [tool_result]}, + ) + return self._turn(key, model, history, "offered") + + +@dataclass(frozen=True, slots=True) +class ResponsesSurface: + sdk: SdkClients + name: SurfaceName = "responses" + + def _turn(self, key: str, model: str, history: ResponseInputParam, tool: ToolMode) -> Reply: + raw: Final = self.sdk.openai(key).responses.with_raw_response.create( + model=model, + input=history, + instructions=INSTRUCTIONS, + max_output_tokens=MAX_OUTPUT_TOKENS, + tools=openai.omit if tool == "none" else [_responses_tool()], + tool_choice=_responses_tool_choice() if tool == "forced" else openai.omit, + parallel_tool_calls=False if tool == "forced" else openai.omit, + extra_body=NO_PROXY_CACHE, + ) + response: Final = raw.parse() + return Reply( + response_id=response.id, + text=response.output_text, + tool_calls=tuple( + ToolCall(call_id=item.call_id, name=item.name, arguments=item.arguments) + for item in response.output + if isinstance(item, ResponseFunctionToolCall) + ), + usage=None if response.usage is None else _usage(response.usage.input_tokens, response.usage.output_tokens), + call_id_header=response_header(raw.headers, "x-litellm-call-id"), + cost_header=response_header(raw.headers, "x-litellm-response-cost"), + ) + + def reply(self, key: str, model: str, prompt: str, *, with_tool: bool = False) -> Reply: + return self._turn(key, model, [{"role": "user", "content": prompt}], "forced" if with_tool else "none") + + def stream(self, key: str, model: str, prompt: str) -> StreamedReply: + events: Final[tuple[ResponseStreamEvent, ...]] = tuple( + self.sdk.openai(key).responses.create( + model=model, + input=prompt, + instructions=INSTRUCTIONS, + max_output_tokens=MAX_OUTPUT_TOKENS, + stream=True, + extra_body=NO_PROXY_CACHE, + ) + ) + return StreamedReply( + text="".join(event.delta for event in events if event.type == "response.output_text.delta"), + finished=bool(events) and events[-1].type == "response.completed", + usage_reported=any( + event.type == "response.completed" and event.response.usage is not None for event in events + ), + event_count=len(events), + ) + + def reply_to_tool_result(self, key: str, model: str, prompt: str, call: ToolCall, result: str) -> Reply: + function_call: Final[ResponseFunctionToolCallParam] = { + "type": "function_call", + "call_id": call.call_id, + "name": call.name, + "arguments": call.arguments, + } + output: Final[FunctionCallOutput] = { + "type": "function_call_output", + "call_id": call.call_id, + "output": result, + } + return self._turn(key, model, [{"role": "user", "content": prompt}, function_call, output], "offered") + + +def build_surfaces(sdk: SdkClients) -> Mapping[SurfaceName, Surface]: + return MappingProxyType[SurfaceName, Surface]( + { + "chat_completions": ChatCompletionsSurface(sdk), + "messages": MessagesSurface(sdk), + "responses": ResponsesSurface(sdk), + } + ) diff --git a/tests/e2e/llm_translation/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py index 7c4a9cc4af9..7da7ceac4d3 100644 --- a/tests/e2e/llm_translation/realtime/realtime_client.py +++ b/tests/e2e/llm_translation/realtime/realtime_client.py @@ -286,7 +286,7 @@ def function_call_item(events: tuple[ReceivedEvent, ...]) -> OutputItem | None: # ---- session + client -------------------------------------------------- -def _as_text(message: str | bytes) -> str: +def as_text(message: str | bytes) -> str: return message.decode("utf-8") if isinstance(message, bytes) else message @@ -304,7 +304,7 @@ class RealtimeSession: collected: list[ReceivedEvent] = [] while time.monotonic() < deadline: try: - text = _as_text( + text = as_text( self.connection.recv(timeout=deadline - time.monotonic()) ) except TimeoutError: diff --git a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py index f99fa8d86b3..d7870b26497 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py @@ -13,8 +13,9 @@ failure. See REALTIME_COVERAGE_MATRIX.md. """ import pytest +from lifecycle import ResourceManager +from models import LiteLLMParamsBody from pydantic import BaseModel - from realtime_client import ( PROVIDERS, ConversationItemCreate, @@ -27,14 +28,17 @@ from realtime_client import ( RealtimeProvider, ResponseCreate, ResponseDone, + ServerEnvelope, SessionConfig, SessionUpdate, + as_text, function_call_item, parse_last, realtime_model, transcript, user_message, ) +from websockets.exceptions import ConnectionClosedError pytestmark = pytest.mark.e2e @@ -147,3 +151,39 @@ def test_tool_call_round_trip( second = session.collect_until("response.done", timeout=60) assert "72" in transcript(second), "follow-up did not use the tool result" + + +_REFUSED_UPSTREAMS = ( + RealtimeProvider( + "azure-bad-key", + "azure-realtime-refused", + LiteLLMParamsBody( + model="azure/gpt-realtime", + api_key="invalid-e2e-key", + api_version="2025-08-28", + realtime_protocol="GA", + ), + ), +) + + +@pytest.mark.parametrize("provider", _REFUSED_UPSTREAMS, ids=[p.id for p in _REFUSED_UPSTREAMS]) +def test_upstream_handshake_refusal_is_an_error_event_and_policy_close( + client: RealtimeClient, + resources: ResourceManager, + scoped_key: str, + provider: RealtimeProvider, +) -> None: + model_name, model_id = client.provision(provider) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + with client.connect(key=scoped_key, model=model_name) as session: + first = ServerEnvelope.model_validate_json( + as_text(session.connection.recv(timeout=15)) + ) + assert first.type == "error", first + with pytest.raises(ConnectionClosedError) as closed: + session.connection.recv(timeout=15) + + assert closed.value.rcvd is not None + assert closed.value.rcvd.code == 1008, closed.value diff --git a/tests/e2e/llm_translation/test_conversational_matrix_e2e.py b/tests/e2e/llm_translation/test_conversational_matrix_e2e.py new file mode 100644 index 00000000000..a5cfc9ce70b --- /dev/null +++ b/tests/e2e/llm_translation/test_conversational_matrix_e2e.py @@ -0,0 +1,163 @@ +"""The same conversation contract on every (endpoint, deployment, auth) cell. +A deployment is one provider model (openai/gpt-4o-mini, anthropic/claude-haiku-4-5, ...). + +/chat/completions, /v1/messages and /v1/responses each have their own +translation code in the proxy, so a bug fixed on one surface tends to survive +on the others. Every test here runs once per cell in `CELLS` +(conversational_matrix.py), so a change to a shared helper is proven against all +surfaces and providers at once, and a new model or provider is one row in `DEPLOYMENTS`. + +Edge-wired: OpenAI and Anthropic traffic goes through the provider edge in +record and replay, so the whole matrix replays with zero provider calls. +""" + +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from typing import Final + +import pytest +from e2e_config import unique_marker +from lifecycle import ResourceManager +from llm_translation.conversational_matrix import ( + GREETING_PROMPT, + WEATHER_PROMPT, + WEATHER_REPORT, + WEATHER_TOOL_NAME, + Cell, + Deployments, + Surface, + SurfaceName, + ToolCall, + build_surfaces, + cells_covering, + register_deployments, +) +from llm_translation.sdk_clients import SdkClients +from models import SpendLogRow +from proxy_client import ProxyClient + +pytestmark = [pytest.mark.e2e, pytest.mark.replayable] + + +def _approx_equal(actual: float, expected: float) -> bool: + return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) + + +@pytest.fixture(scope="module") +def deployments(proxy: ProxyClient) -> Iterator[Deployments]: + yield from register_deployments(proxy) + + +@pytest.fixture(scope="module") +def surfaces(sdk: SdkClients) -> Mapping[SurfaceName, Surface]: + return build_surfaces(sdk) + + +def _weather_call(surface: Surface, key: str, model: str) -> ToolCall: + first: Final = surface.reply(key, model, WEATHER_PROMPT, with_tool=True) + assert len(first.tool_calls) == 1, ( + f"{surface.name} forced tool_choice={WEATHER_TOOL_NAME} with parallel calls off, " + f"got {len(first.tool_calls)} tool call(s): {first.tool_calls} text={first.text!r}" + ) + call: Final = first.tool_calls[0] + assert call.name == WEATHER_TOOL_NAME, f"{surface.name} called {call.name!r}, not the forced {WEATHER_TOOL_NAME!r}" + assert call.call_id, f"{surface.name} tool call has no id, so the caller cannot answer it: {call}" + assert "paris" in call.parsed().location.lower(), f"{surface.name} tool arguments lost the location: {call}" + return call + + +class TestConversationalMatrix: + @pytest.mark.parametrize("cell", cells_covering("basic", "nonstream", "works")) + def test_reply_carries_assistant_text_and_usage( + self, + cell: Cell, + deployments: Deployments, + surfaces: Mapping[SurfaceName, Surface], + resources: ResourceManager, + ) -> None: + surface: Final = surfaces[cell.surface] + reply: Final = surface.reply(resources.key(), deployments.alias(cell), GREETING_PROMPT) + + assert reply.response_id, f"{cell.id}: response has no id" + assert reply.text.strip(), f"{cell.id}: response carried no assistant text" + assert reply.usage is not None and reply.usage.input_tokens > 0 and reply.usage.output_tokens > 0, ( + f"{cell.id}: usage missing or zero, so the caller cannot account for this call: {reply.usage}" + ) + assert reply.call_id_header, f"{cell.id}: x-litellm-call-id header missing" + + @pytest.mark.parametrize("cell", cells_covering("basic", "stream", "works")) + def test_stream_delivers_text_usage_and_a_terminal_event( + self, + cell: Cell, + deployments: Deployments, + surfaces: Mapping[SurfaceName, Surface], + resources: ResourceManager, + ) -> None: + surface: Final = surfaces[cell.surface] + streamed: Final = surface.stream(resources.key(), deployments.alias(cell), GREETING_PROMPT) + + assert streamed.event_count > 1, f"{cell.id}: stream arrived as {streamed.event_count} event(s), not a stream" + assert streamed.text.strip(), f"{cell.id}: stream carried no text deltas" + assert streamed.finished, f"{cell.id}: stream never sent its terminal event" + assert streamed.usage_reported, f"{cell.id}: stream never reported usage" + + @pytest.mark.parametrize("cell", cells_covering("basic", "nonstream", "cost_logged")) + def test_cost_header_matches_the_spend_log( + self, + cell: Cell, + deployments: Deployments, + surfaces: Mapping[SurfaceName, Surface], + resources: ResourceManager, + proxy: ProxyClient, + ) -> None: + key: Final = resources.key() + surface: Final = surfaces[cell.surface] + reply: Final = surface.reply(key, deployments.alias(cell), f"{GREETING_PROMPT} {unique_marker()}") + + assert reply.cost_header is not None, f"{cell.id}: x-litellm-response-cost header missing" + header_cost: Final = float(reply.cost_header) + assert header_cost > 0, f"{cell.id}: x-litellm-response-cost is not positive: {header_cost}" + + def _priced(rows: list[SpendLogRow]) -> bool: + return any(row.spend is not None and row.spend > 0 for row in rows) + + rows: Final = proxy.poll_logs_for_key(key, predicate=_priced) + priced: Final = tuple(row for row in rows if row.spend is not None and row.spend > 0) + assert len(priced) == 1, f"{cell.id}: expected exactly one priced spend row for a fresh key, got {rows}" + row: Final = priced[0] + assert (row.prompt_tokens or 0) > 0 and (row.completion_tokens or 0) > 0, ( + f"{cell.id}: spend row has no token counts, so the cost is not real usage: {row}" + ) + assert row.spend is not None and _approx_equal(row.spend, header_cost), ( + f"{cell.id}: logged spend {row.spend} disagrees with x-litellm-response-cost {header_cost}" + ) + assert row.model and cell.deployment.backend.endswith(row.model), ( + f"{cell.id}: spend row logged model {row.model!r}, not the deployment's {cell.deployment.backend!r}" + ) + + @pytest.mark.parametrize("cell", cells_covering("tool_use", "nonstream", "works")) + def test_tool_call_is_returned_named_and_addressable( + self, + cell: Cell, + deployments: Deployments, + surfaces: Mapping[SurfaceName, Surface], + resources: ResourceManager, + ) -> None: + _weather_call(surfaces[cell.surface], resources.key(), deployments.alias(cell)) + + @pytest.mark.parametrize("cell", cells_covering("multi_turn", "nonstream", "works")) + def test_tool_result_round_trip_reaches_the_model( + self, + cell: Cell, + deployments: Deployments, + surfaces: Mapping[SurfaceName, Surface], + resources: ResourceManager, + ) -> None: + key: Final = resources.key() + model: Final = deployments.alias(cell) + surface: Final = surfaces[cell.surface] + call: Final = _weather_call(surface, key, model) + + answer: Final = surface.reply_to_tool_result(key, model, WEATHER_PROMPT, call, WEATHER_REPORT) + assert "22" in answer.text, f"{cell.id}: the model never saw the tool result: {answer.text!r}" diff --git a/tests/e2e/llm_translation/test_credential_messages_e2e.py b/tests/e2e/llm_translation/test_credential_messages_e2e.py index 52306ce3a7a..58e17f20bb8 100644 --- a/tests/e2e/llm_translation/test_credential_messages_e2e.py +++ b/tests/e2e/llm_translation/test_credential_messages_e2e.py @@ -49,5 +49,6 @@ class TestCredentialBackedMessages: extra_body=NO_PROXY_CACHE, ) assert message.role == "assistant", f"unexpected role: {message.role!r}" + assert message.usage.output_tokens > 0, f"/v1/messages billed no output tokens: {message.usage!r}" text = "".join(block.text for block in message.content if block.type == "text") assert text.strip(), f"/v1/messages returned no text: {message.content!r}" diff --git a/tests/e2e/llm_translation/test_xiaomi_mimo_e2e.py b/tests/e2e/llm_translation/test_xiaomi_mimo_e2e.py deleted file mode 100644 index efca216634b..00000000000 --- a/tests/e2e/llm_translation/test_xiaomi_mimo_e2e.py +++ /dev/null @@ -1,214 +0,0 @@ -"""Live e2e: Xiaomi MiMo v2.6 through the gateway on /chat/completions. - -Both native ``xiaomi_mimo/`` v2.6 rows (pro and flash) are registered via -``/model/new`` and driven against Xiaomi's own endpoint. What the gateway owes -us is that the reasoning chain surfaces as ``reasoning_content``, tool calls -survive translation, and the cost header plus spend row follow the proxy's own -cost-map price for the row (read back from ``/model/info``, never pinned here). -Requires XIAOMI_MIMO_API_KEY on the proxy; no skip gate. -""" - -from __future__ import annotations - -from typing import Final - -import pytest -from e2e_config import unique_marker -from e2e_http import StreamingResponse, require_successful_call, unwrap -from lifecycle import ResourceManager -from models import ( - ChatBody, - ChatMessage, - ChatResponse, - ChatTool, - ChatToolFunction, - CostMapEntry, - LiteLLMParamsBody, - OutMessage, - SpendLogRow, -) -from passthrough_client import PassthroughClient -from pydantic import BaseModel - -pytestmark = pytest.mark.e2e - -BACKENDS: Final = ("xiaomi_mimo/mimo-v2.6-pro", "xiaomi_mimo/mimo-v2.6-flash") -ARITHMETIC_PROMPT = "What is 17 + 26? Answer with just the number." -WEATHER_PROMPT = "What is the weather in Paris? Use the tool." -COUNTING_PROMPT = "Count from 1 to 50, one number per line." - -WEATHER_TOOL = ChatTool( - function=ChatToolFunction( - name="get_weather", - description="Get the current weather for a location.", - parameters={ - "type": "object", - "properties": {"location": {"type": "string"}}, - "required": ["location"], - }, - ) -) - - -class _WeatherArgs(BaseModel): - location: str - - -class _StreamDelta(BaseModel): - content: str | None = None - reasoning_content: str | None = None - - -class _StreamChoice(BaseModel): - delta: _StreamDelta | None = None - - -class _StreamChunk(BaseModel): - choices: list[_StreamChoice] = [] - - -def _approx_equal(actual: float, expected: float) -> bool: - return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) - - -@pytest.fixture(scope="module") -def registry(client: PassthroughClient) -> dict[str, CostMapEntry]: - return client.proxy.model_cost_map() - - -def _register(client: PassthroughClient, resources: ResourceManager, backend: str) -> tuple[str, str]: - model = f"e2e-xiaomi-{unique_marker()}" - model_id = client.proxy.create_model( - model, LiteLLMParamsBody(model=backend, api_key="os.environ/XIAOMI_MIMO_API_KEY") - ) - resources.defer(lambda: client.proxy.delete_model(model_id)) - return model, resources.key() - - -def _message(response: ChatResponse) -> OutMessage: - assert response.choices, f"Xiaomi returned no choices: {response}" - message = response.choices[0].message - assert message is not None, f"Xiaomi choice has no message: {response}" - return message - - -def _deltas(result: StreamingResponse) -> list[_StreamDelta]: - require_successful_call(result) - assert result.is_streaming, f"response was not streamed: {result.headers}" - assert not result.stream_error, f"stream errored: {result.stream_error}" - assert result.stream_done, f"stream never reached [DONE]: {result.stream_events[-3:]}" - return [ - choice.delta - for event in result.stream_events - for choice in _StreamChunk.model_validate_json(event).choices - if choice.delta is not None - ] - - -@pytest.mark.parametrize("backend", BACKENDS) -class TestXiaomiMimoChatCompletions: - @pytest.mark.covers("llm.chat_completions.xiaomi_mimo.basic.nonstream.cost_logged") - def test_cost_header_and_spend_row_match_the_registry_price( - self, - client: PassthroughClient, - resources: ResourceManager, - registry: dict[str, CostMapEntry], - backend: str, - ) -> None: - price = registry.get(backend) - assert price is not None, f"{backend} has no row in the proxy's cost map, so native calls would bill $0" - assert price.litellm_provider == "xiaomi_mimo", f"{backend} is filed under the wrong provider: {price}" - assert price.input_cost_per_token and price.output_cost_per_token, f"{backend} carries no price: {price}" - model, key = _register(client, resources, backend) - - result = client.proxy.transport.send( - "/chat/completions", - headers=client.proxy.transport.bearer(key), - json=ChatBody( - model=model, - messages=[ChatMessage(role="user", content=f"{ARITHMETIC_PROMPT} {unique_marker()}")], - max_tokens=1024, - ), - ) - require_successful_call(result) - response = ChatResponse.model_validate_json(result.body) - message = _message(response) - assert message.content and "43" in message.content, f"answer lost: {message}" - assert message.reasoning_content, f"{backend} reasons, but no reasoning_content came back: {message}" - - usage = response.usage - assert usage is not None and usage.prompt_tokens and usage.completion_tokens, ( - f"response carries no usage, so the cost cannot be real: {result.body[:300]}" - ) - header_cost = result.response_cost - assert header_cost is not None and header_cost > 0, ( - f"x-litellm-response-cost header missing or non-positive: {result.headers}" - ) - cached = (usage.prompt_tokens_details.cached_tokens or 0) if usage.prompt_tokens_details else 0 - expected = ( - (usage.prompt_tokens - cached) * price.input_cost_per_token - + cached * (price.cache_read_input_token_cost or 0.0) - + usage.completion_tokens * price.output_cost_per_token - ) - assert _approx_equal(header_cost, expected), ( - f"header cost {header_cost} disagrees with the registry price for {backend} at {usage}: expected {expected}" - ) - - def _priced(rows: list[SpendLogRow]) -> bool: - return any(row.spend is not None and row.spend > 0 for row in rows) - - rows = client.proxy.poll_logs_for_key(key, predicate=_priced) - priced = [row for row in rows if row.spend is not None and row.spend > 0] - assert priced, f"no priced spend row landed for key {key}; got {rows}" - row = priced[0] - assert row.custom_llm_provider == "xiaomi_mimo", f"spend row misattributed: {row}" - assert row.spend is not None and _approx_equal(row.spend, header_cost), ( - f"logged spend {row.spend} disagrees with the x-litellm-response-cost header {header_cost}" - ) - - @pytest.mark.covers("llm.chat_completions.xiaomi_mimo.thinking.stream.works") - def test_reasoning_and_answer_stream_as_deltas( - self, client: PassthroughClient, resources: ResourceManager, backend: str - ) -> None: - model, key = _register(client, resources, backend) - - deltas = _deltas( - client.proxy.chat_stream( - key, - ChatBody( - model=model, - messages=[ChatMessage(role="user", content=COUNTING_PROMPT)], - max_tokens=2048, - stream=True, - ), - ) - ) - reasoning = "".join(delta.reasoning_content or "" for delta in deltas) - content = "".join(delta.content or "" for delta in deltas) - assert reasoning, f"stream carried no reasoning_content deltas: {deltas[:5]}" - assert "50" in content, f"streamed answer lost: {content[:300]!r}" - - @pytest.mark.covers("llm.chat_completions.xiaomi_mimo.tool_use.nonstream.works") - def test_tool_call_is_returned(self, client: PassthroughClient, resources: ResourceManager, backend: str) -> None: - model, key = _register(client, resources, backend) - - message = _message( - unwrap( - client.proxy.chat( - key, - ChatBody( - model=model, - messages=[ChatMessage(role="user", content=WEATHER_PROMPT)], - tools=[WEATHER_TOOL], - max_tokens=1024, - ), - ) - ) - ) - assert message.tool_calls, f"{backend} dropped the tool call: {message}" - call = message.tool_calls[0] - assert call.id, f"tool call carries no id, so a tool result cannot answer it: {call}" - assert call.function.name == "get_weather", f"wrong tool called: {call}" - assert call.function.arguments, f"tool call carries no arguments: {call}" - args = _WeatherArgs.model_validate_json(call.function.arguments) - assert "paris" in args.location.lower(), f"tool arguments lost the location: {args}" diff --git a/tests/e2e/logging/logging_client.py b/tests/e2e/logging/logging_client.py index 66dfa233ec4..c2f987ea33d 100644 --- a/tests/e2e/logging/logging_client.py +++ b/tests/e2e/logging/logging_client.py @@ -146,6 +146,29 @@ class LangfuseObservationList(BaseModel): data: list[LangfuseObservation] = [] +class LangfuseOtelMetadata(BaseModel): + """Langfuse stores every OTel span attribute under metadata.attributes.""" + + model_config = ConfigDict(extra="ignore") + + attributes: dict[str, str] = {} + + +def otel_attributes(obs: LangfuseObservation) -> dict[str, str]: + try: + return LangfuseOtelMetadata.model_validate(obs.metadata).attributes + except ValidationError: + return {} + + +def is_otel_v2_generation(obs: LangfuseObservation, *, key_alias: str) -> bool: + attributes = otel_attributes(obs) + return ( + attributes.get("langfuse.observation.type") == "generation" + and attributes.get("litellm.metadata.user_api_key_alias") == key_alias + ) + + class LangfuseListParams(BaseModel): model_config = ConfigDict(populate_by_name=True) @@ -630,6 +653,18 @@ class LoggingClient: time.sleep(POLL_INTERVAL) return last + def poll_langfuse_generation( + self, creds: LangfuseCreds, *, key_alias: str, from_start_time: str + ) -> LangfuseObservation | None: + """The OTel v2 generation the proxy exported for one key alias since from_start_time.""" + deadline = time.monotonic() + POLL_TIMEOUT + while time.monotonic() < deadline: + for obs in self.list_langfuse_observations(creds, from_start_time=from_start_time): + if is_otel_v2_generation(obs, key_alias=key_alias): + return obs + time.sleep(POLL_INTERVAL) + return None + def poll_langfuse_trace_observations( self, creds: LangfuseCreds, diff --git a/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py b/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py new file mode 100644 index 00000000000..874b2b6a045 --- /dev/null +++ b/tests/e2e/logging/test_langsmith_batch_serialization_e2e.py @@ -0,0 +1,125 @@ +"""Live e2e: a LangSmith batch whose metadata holds non JSON-native Python values +(datetime, Decimal) must reach the real LangSmith API instead of dying in +json.dumps and dropping the whole batch. Only the SDK path can put such values +into the batch (the proxy JSON-decodes request metadata), so this test drives +litellm.acompletion in-process against the real OpenAI API with a LangsmithLogger +injected per request, flushes the batch, and reads the run back by id through +LangSmith's own API. Nothing is mocked. +""" + +from __future__ import annotations + +import asyncio +import datetime +import decimal +import os +import time +import uuid +from dataclasses import dataclass +from typing import Final + +import pytest +from e2e_config import CHEAP_OPENAI_MODEL, POLL_INTERVAL, POLL_TIMEOUT, unique_marker +from e2e_http import Headers, Success, get_external +from pydantic import BaseModel, ConfigDict, Field, JsonValue + +import litellm +from litellm.integrations.langsmith import LangsmithLogger + +pytestmark = pytest.mark.e2e + + +class LangsmithHeaders(Headers): + x_api_key: str = Field(serialization_alias="x-api-key") + + +class LangsmithRunExtra(BaseModel): + model_config = ConfigDict(extra="allow") + requester_metadata: dict[str, JsonValue] | None = None + + +class LangsmithRun(BaseModel): + id: str + session_name: str | None = None + extra: LangsmithRunExtra + + +@dataclass(frozen=True, slots=True) +class LangsmithCreds: + api_key: str + base_url: str + project: str + + +def load_langsmith_creds() -> LangsmithCreds: + api_key = os.getenv("LANGSMITH_API_KEY") + if not api_key: + pytest.fail("LangSmith e2e requires LANGSMITH_API_KEY; missing credentials is a hard failure, not a skip") + if os.getenv("LANGSMITH_MOCK"): + pytest.fail("LANGSMITH_MOCK is set; this e2e must hit the real LangSmith API") + return LangsmithCreds( + api_key=api_key, + base_url=(os.getenv("LANGSMITH_BASE_URL") or "https://api.smith.langchain.com").rstrip("/"), + project=os.getenv("LANGSMITH_PROJECT") or "litellm-e2e", + ) + + +def _fetch_run(creds: LangsmithCreds, run_id: uuid.UUID) -> LangsmithRun | None: + result = get_external( + f"{creds.base_url}/runs/{run_id}", + response_type=LangsmithRun, + headers=LangsmithHeaders(x_api_key=creds.api_key), + ) + match result: + case Success(data=run): + return run + case _: + return None + + +def _poll_run(creds: LangsmithCreds, run_id: uuid.UUID) -> LangsmithRun: + deadline: Final = time.monotonic() + POLL_TIMEOUT + while time.monotonic() < deadline: + run = _fetch_run(creds, run_id) + if run is not None: + return run + time.sleep(POLL_INTERVAL) + pytest.fail(f"LangSmith run {run_id} never appeared within {POLL_TIMEOUT}s; the batch flush dropped it") + + +class TestLangsmithBatchSerialization: + @pytest.mark.asyncio + @pytest.mark.covers("logging.langsmith.success.serializes_non_native_metadata") + async def test_non_json_native_metadata_reaches_langsmith(self) -> None: + creds: Final = load_langsmith_creds() + logger: Final = LangsmithLogger( + langsmith_api_key=creds.api_key, langsmith_project=creds.project, langsmith_base_url=creds.base_url + ) + assert not logger.is_mock_mode, "LangsmithLogger initialised in mock mode; this e2e needs the real API" + marker: Final = unique_marker() + run_id: Final = uuid.uuid4() + created_at: Final = datetime.datetime(2026, 1, 2, 3, 4, 5, tzinfo=datetime.timezone.utc) + spend: Final = decimal.Decimal("0.0042") + response: Final = await litellm.acompletion( + model=f"openai/{CHEAP_OPENAI_MODEL}", + messages=[{"role": "user", "content": f"Reply with the single word ok ({marker})"}], + max_completion_tokens=5, + callbacks=[logger], + metadata={"run_id": str(run_id), "metadata": {"marker": marker, "created_at": created_at, "spend": spend}}, + ) + assert isinstance(response, litellm.ModelResponse) and response.id, ( + "a non-streaming completion must return a ModelResponse before the batch flush is meaningful" + ) + enqueue_deadline: Final = time.monotonic() + POLL_TIMEOUT + while time.monotonic() < enqueue_deadline and len(logger.log_queue) == 0: + await asyncio.sleep(0.5) + assert len(logger.log_queue) == 1, ( + f"the completion must be queued for the LangSmith batch, got {len(logger.log_queue)} queued entries" + ) + await logger.async_send_batch() + run: Final = _poll_run(creds, run_id) + requester_metadata: Final = run.extra.requester_metadata + assert requester_metadata is not None, "the run must carry the caller metadata under extra.requester_metadata" + assert requester_metadata["marker"] == marker + assert requester_metadata["created_at"] == str(created_at) + assert requester_metadata["spend"] == str(spend) diff --git a/tests/e2e/logging/test_otel_trace_e2e.py b/tests/e2e/logging/test_otel_trace_e2e.py index 9f08fa6c4e7..8d154ca0837 100644 --- a/tests/e2e/logging/test_otel_trace_e2e.py +++ b/tests/e2e/logging/test_otel_trace_e2e.py @@ -12,21 +12,24 @@ commit 1bd603d1ac). Both halves of the contract are asserted: the recorded state (the proxy reports the OTEL v2 logger active via /health/readiness/details) and the enforced behavior (the complete span tree at the destination, read back through the -destination's own query API - never proxy-side "export succeeded" logs). +destination's own query API - never proxy-side "export succeeded" logs). The +TLS coverage requires the stack to export OTLP over HTTPS with a certificate +signed by the CA in SSL_CERT_FILE, and treats a missing or plaintext endpoint +as a stack misconfiguration rather than skipping the test. """ from __future__ import annotations import time +from typing import Final import pytest -from pydantic import BaseModel, ConfigDict, ValidationError - -from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, unique_marker +from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, OTEL_EXPORTER_ENDPOINT, unique_marker from lifecycle import ResourceManager from logging_client import INVALID_UPSTREAM_API_KEY, LoggingClient, first_ok, readiness_details_body from models import LiteLLMParamsBody from otel_client import JaegerSpan, JaegerTrace, OtelReader +from pydantic import BaseModel, ConfigDict, ValidationError pytestmark = pytest.mark.e2e @@ -312,6 +315,35 @@ class TestOtelTraceCompleteness: ) _assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}") + @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"]) + @pytest.mark.otel_tls + def test_otel_export_over_tls_with_internal_ca_reaches_destination( + self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager + ) -> None: + _assert_otel_destination_configured(client) + assert OTEL_EXPORTER_ENDPOINT.startswith("https://"), ( + "the stack must export OTLP over TLS signed by the CA in SSL_CERT_FILE " + "(E2E_OTEL_EXPORTER_ENDPOINT) for this test to prove anything; a " + "missing or plaintext value is a stack misconfiguration" + ) + + route: Final = "/chat/completions" + key: Final = client.key_with_alias(f"otel-trace-tls-{unique_marker()}", models=[MODEL]) + resources.defer(lambda: client.delete_key(key)) + + marker: Final = unique_marker() + outcome: Final = first_ok( + client, lambda: client.chat_raw(key, MODEL, f"reply with one word {marker}", max_tokens=16) + ) + assert outcome.call_id is not None, "success response must carry x-litellm-call-id" + + hits: Final = otel_reader.poll_traces_for_call( + call_id=outcome.call_id, + settled_names=_settled_names(route=route, genai_span=f"chat {MODEL}"), + settled_prefixes={DB_SPAN_PREFIX}, + ) + _assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}") + @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"]) def test_messages_exports_complete_trace( self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager diff --git a/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py b/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py new file mode 100644 index 00000000000..9e08cf21da0 --- /dev/null +++ b/tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py @@ -0,0 +1,415 @@ +"""Live e2e: the OTel v2 Langfuse generation carries input and output for every non-chat endpoint. + +With LITELLM_OTEL_V2=true the proxy exports one generation per request to the +team's Langfuse destination. Chat, Responses and embeddings already fill both +panels; this file pins the other families. LIT-8309 covers the output of +completions, images, speech, transcription and moderation; LIT-8326 covers the +rerank and search output and the OCR, image-edit and search input, which used +to read the `default-message-value` placeholder. Each test registers a real +deployment, drives the endpoint through the shared transport, then reads the +generation back from Langfuse and asserts it reflects what the caller sent and +received: the completion text, the transcript, the moderation verdict, the +ranked rerank indices and scores, the search results, the OCR document URL, +the edit prompt, and for images, speech and uploaded documents a bounded +summary that never carries the raw base64 or audio bytes. +""" + +from __future__ import annotations + +import base64 +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Final + +import pytest +from e2e_config import unique_marker +from e2e_http import unwrap +from lifecycle import ResourceManager +from logging_client import LangfuseCreds, LangfuseObservation, LoggingClient, load_langfuse_creds +from models import ( + CompletionBody, + CompletionResponse, + ImageEditForm, + ImageGenerationBody, + ImageGenerationResponse, + LiteLLMParamsBody, + ModerationBody, + ModerationResponse, + OcrBody, + OcrDocument, + OcrForm, + OcrResponse, + RerankBody, + SearchBody, + SearchResponse, + SearchToolBody, + SearchToolCreateBody, + SearchToolLiteLLMParamsBody, + SpeechBody, + TranscriptionForm, + TranscriptionResponse, +) +from pydantic import BaseModel, TypeAdapter, ValidationError + +pytestmark = [pytest.mark.e2e, pytest.mark.otel_v2] + +WEATHER_WAV: Final = ( + Path(__file__).resolve().parent.parent / "llm_translation" / "realtime" / "fixtures" / "weather_question_24k.wav" +) +DUMMY_PDF: Final = Path(__file__).resolve().parent.parent.parent / "llm_translation" / "fixtures" / "dummy.pdf" +DUMMY_PDF_URL: Final = ( + "https://cdn.jsdelivr.net/gh/BerriAI/litellm" + "@d769e81c90d453240c61fc572cdb27fae06a89d0" + "/tests/llm_translation/fixtures/dummy.pdf" +) +RED_SQUARE_PNG: Final = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PMQ0AAAwDoPo3" + "3UrYvQQckD4XAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB" + "AYHLAMpT0sIcNbcEAAAAAElFTkSuQmCC" +) +BOUNDED_OUTPUT_CHARS: Final = 1024 +PLACEHOLDER_INPUT: Final = "default-message-value" + + +class _OutputMessage(BaseModel): + """One assistant message of the Langfuse generation output; only the text is read.""" + + content: str = "" + + +class _InputMessage(BaseModel): + """One user message of the Langfuse generation input; only the text is read.""" + + content: str = "" + + +_OUTPUT_MESSAGES: Final = TypeAdapter(list[_OutputMessage]) +_INPUT_MESSAGES: Final = TypeAdapter(list[_InputMessage]) + + +@pytest.fixture(scope="session") +def langfuse_creds() -> LangfuseCreds: + return load_langfuse_creds() + + +def _langfuse_key( + client: LoggingClient, creds: LangfuseCreds, resources: ResourceManager, params: LiteLLMParamsBody +) -> tuple[str, str, str]: + """A model registered for this run plus a key on a team whose Langfuse callback is `creds`.""" + model: Final = f"e2e-otel-out-{unique_marker()}" + model_id: Final = client.proxy.create_model(model, params) + resources.defer(lambda: client.proxy.delete_model(model_id)) + team_id: Final = client.create_team(f"otel-out-team-{unique_marker()}", models=[model]) + resources.defer(lambda: client.delete_team(team_id)) + client.add_team_langfuse_callback(team_id, creds) + alias: Final = f"otel-out-key-{unique_marker()}" + key: Final = client.key_with_alias(alias, models=[model], team_id=team_id) + resources.defer(lambda: client.delete_key(key)) + return model, key, alias + + +def _generation(client: LoggingClient, creds: LangfuseCreds, *, alias: str, started: datetime) -> LangfuseObservation: + since: Final = (started - timedelta(seconds=5)).isoformat() + observation: Final = client.poll_langfuse_generation(creds, key_alias=alias, from_start_time=since) + assert observation is not None, f"no OTel v2 generation reached Langfuse for key alias {alias!r}" + return observation + + +def _output_text(observation: LangfuseObservation) -> str: + assert observation.output not in (None, "", [], {}), f"generation output is empty: {observation!r}" + try: + messages: Final = _OUTPUT_MESSAGES.validate_python(observation.output) + except ValidationError: + pytest.fail(f"generation output is not a list of assistant messages: {observation!r}") + assert messages, f"generation output is empty: {observation!r}" + return "\n".join(message.content for message in messages) + + +def _input_text(observation: LangfuseObservation) -> str: + assert observation.input not in (None, "", [], {}), f"generation input is empty: {observation!r}" + try: + messages: Final = _INPUT_MESSAGES.validate_python(observation.input) + except ValidationError: + pytest.fail(f"generation input is not a list of user messages: {observation!r}") + assert messages, f"generation input is empty: {observation!r}" + text: Final = "\n".join(message.content for message in messages) + assert text != PLACEHOLDER_INPUT, f"generation input is the placeholder, not the request: {observation!r}" + return text + + +def _openai(model: str) -> LiteLLMParamsBody: + return LiteLLMParamsBody(model=model, api_key="os.environ/OPENAI_API_KEY") + + +def _mistral_ocr() -> LiteLLMParamsBody: + return LiteLLMParamsBody(model="mistral/mistral-ocr-latest", api_key="os.environ/MISTRAL_API_KEY") + + +def _langfuse_search_tool( + client: LoggingClient, creds: LangfuseCreds, resources: ResourceManager +) -> tuple[str, str, str]: + """A keyless DuckDuckGo search tool registered for this run, plus a key on a team whose Langfuse callback is + `creds`.""" + tool: Final = f"e2e-otel-search-{unique_marker()}" + tool_id: Final = client.proxy.create_search_tool( + SearchToolCreateBody( + search_tool=SearchToolBody( + search_tool_name=tool, + litellm_params=SearchToolLiteLLMParamsBody(search_provider="duckduckgo"), + ) + ) + ) + resources.defer(lambda: client.proxy.delete_search_tool(tool_id)) + team_id: Final = client.create_team(f"otel-search-team-{unique_marker()}", models=[tool]) + resources.defer(lambda: client.delete_team(team_id)) + client.add_team_langfuse_callback(team_id, creds) + alias: Final = f"otel-search-key-{unique_marker()}" + key: Final = client.key_with_alias(alias, models=[tool], team_id=team_id) + resources.defer(lambda: client.delete_key(key)) + return tool, key, alias + + +class TestOtelV2LangfuseGenerationOutput: + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["completions"]) + def test_completions_output_is_the_completion_text( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-3.5-turbo-instruct")) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.post( + "/v1/completions", + headers=client.proxy.transport.bearer(key), + json=CompletionBody(model=model, prompt=f"Repeat exactly: {unique_marker()}", n=2), + response_type=CompletionResponse, + ) + ) + texts: Final = tuple(choice.text.strip() for choice in response.choices) + assert len(texts) == 2 and all(texts), f"/v1/completions returned no text: {response!r}" + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert all(text in output for text in texts), ( + f"generation output lacks the completion texts {texts!r}: {output!r}" + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["images_generations"]) + def test_images_output_is_a_bounded_summary_without_base64( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-image-1-mini")) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.post( + "/v1/images/generations", + headers=client.proxy.transport.bearer(key), + json=ImageGenerationBody(model=model, prompt=f"a plain red square {unique_marker()}"), + response_type=ImageGenerationResponse, + timeout=180.0, + ) + ) + assert response.data, f"/v1/images/generations returned no data: {response!r}" + encoded: Final = response.data[0].b64_json or "" + assert encoded, f"expected a b64_json image from gpt-image-1-mini: {response.data[0].url!r}" + image_bytes: Final = len(base64.b64decode(encoded)) + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert len(output) <= BOUNDED_OUTPUT_CHARS, f"image generation output is not bounded ({len(output)} chars)" + assert encoded[:64] not in output, "image generation output leaks the raw base64 payload" + assert output == f"b64_json image ({image_bytes} bytes)", ( + f"image generation output does not report the {image_bytes} decoded bytes: {output!r}" + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["audio_speech"]) + def test_speech_output_is_a_bounded_summary_without_audio_bytes( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-4o-mini-tts")) + started: Final = datetime.now(timezone.utc) + audio: Final = client.proxy.transport.stream_binary( + "/v1/audio/speech", + headers=client.proxy.transport.bearer(key), + json=SpeechBody(model=model, input=f"hello {unique_marker()}"), + ) + assert audio.ok and audio.total_bytes > 0, f"/v1/audio/speech returned no audio: {audio!r}" + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert len(output) <= BOUNDED_OUTPUT_CHARS, f"speech output is not bounded ({len(output)} chars)" + assert output.endswith(f" ({audio.total_bytes} bytes)"), ( + f"speech output does not report the {audio.total_bytes} audio bytes the caller received: {output!r}" + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["audio_transcriptions"]) + def test_transcription_output_is_the_transcript( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-4o-mini-transcribe")) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.upload( + "/v1/audio/transcriptions", + headers=client.proxy.transport.bearer(key), + form=TranscriptionForm(model=model), + filename=WEATHER_WAV.name, + content=WEATHER_WAV.read_bytes(), + file_content_type="audio/wav", + response_type=TranscriptionResponse, + ) + ) + transcript: Final = response.text.strip() + assert transcript, f"/v1/audio/transcriptions returned no text: {response!r}" + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert transcript in output, f"generation output lacks the transcript {transcript!r}: {output!r}" + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["moderations"]) + def test_moderations_output_is_the_verdict( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/omni-moderation-latest")) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.post( + "/v1/moderations", + headers=client.proxy.transport.bearer(key), + json=ModerationBody(model=model, input=f"I will find you and hurt you badly {unique_marker()}"), + response_type=ModerationResponse, + ) + ) + assert response.results, f"/v1/moderations returned no results: {response!r}" + verdict: Final = "flagged: " if response.results[0].flagged else "not flagged" + + output: Final = _output_text(_generation(client, langfuse_creds, alias=alias, started=started)) + assert output.startswith(verdict), ( + f"generation output does not carry the moderation verdict {verdict!r}: {output!r}" + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["rerank"]) + def test_rerank_output_is_the_ranked_indices_and_scores( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key( + client, + langfuse_creds, + resources, + LiteLLMParamsBody(model="cohere/rerank-v4.0-fast", api_key="os.environ/COHERE_API_KEY"), + ) + query: Final = f"What is the capital of France? {unique_marker()}" + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.rerank( + key, + RerankBody( + model=model, + query=query, + documents=["Paris is the capital of France.", "Berlin is in Germany.", "Bananas are yellow."], + top_n=2, + ), + ) + ) + ranked: Final = tuple(f"[{item.index}] {item.relevance_score}" for item in response.results) + assert len(ranked) == 2 and all(item.index is not None for item in response.results), ( + f"/v1/rerank returned no ranked results: {response!r}" + ) + + generation: Final = _generation(client, langfuse_creds, alias=alias, started=started) + assert _input_text(generation) == query + output: Final = _output_text(generation) + assert output == "\n\n".join(ranked), f"generation output is not the ranked indices and scores: {output!r}" + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["ocr"]) + def test_ocr_input_is_the_document_url( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _mistral_ocr()) + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.ocr( + key, OcrBody(model=model, document=OcrDocument(type="document_url", document_url=DUMMY_PDF_URL)) + ) + ) + assert response.pages and response.pages[0].markdown, f"/v1/ocr returned no page markdown: {response!r}" + + generation: Final = _generation(client, langfuse_creds, alias=alias, started=started) + assert _input_text(generation) == DUMMY_PDF_URL + assert response.pages[0].markdown in _output_text(generation) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["ocr"]) + def test_ocr_upload_input_is_a_bounded_document_summary_without_base64( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _mistral_ocr()) + pdf: Final = DUMMY_PDF.read_bytes() + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.upload( + "/v1/ocr", + headers=client.proxy.transport.bearer(key), + form=OcrForm(model=model), + filename=DUMMY_PDF.name, + content=pdf, + file_content_type="application/pdf", + response_type=OcrResponse, + ) + ) + assert response.pages and response.pages[0].markdown, f"/v1/ocr returned no page markdown: {response!r}" + + text: Final = _input_text(_generation(client, langfuse_creds, alias=alias, started=started)) + encoded: Final = base64.b64encode(pdf).decode() + assert text == f"data:application/pdf;base64 ({len(encoded)} chars)", ( + f"OCR upload input is not the bounded document summary: {text!r}" + ) + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["images_edits"]) + def test_image_edit_input_is_the_edit_prompt( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + model, key, alias = _langfuse_key(client, langfuse_creds, resources, _openai("openai/gpt-image-1-mini")) + prompt: Final = f"make the square blue {unique_marker()}" + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.upload( + "/v1/images/edits", + headers=client.proxy.transport.bearer(key), + form=ImageEditForm(model=model, prompt=prompt), + filename="red_square.png", + content=RED_SQUARE_PNG, + file_content_type="image/png", + file_field="image", + response_type=ImageGenerationResponse, + timeout=180.0, + ) + ) + assert response.data and (response.data[0].b64_json or response.data[0].url), ( + f"/v1/images/edits returned no image: {response!r}" + ) + + generation: Final = _generation(client, langfuse_creds, alias=alias, started=started) + assert _input_text(generation) == prompt + assert _output_text(generation).startswith("b64_json image (") + + @pytest.mark.covers("logging.langfuse.success.logs_spend", exercised_on=["search"]) + def test_search_input_is_the_query_and_output_the_results( + self, client: LoggingClient, langfuse_creds: LangfuseCreds, resources: ResourceManager + ) -> None: + tool, key, alias = _langfuse_search_tool(client, langfuse_creds, resources) + query: Final = "Eiffel Tower" + started: Final = datetime.now(timezone.utc) + response: Final = unwrap( + client.proxy.transport.post( + f"/v1/search/{tool}", + headers=client.proxy.transport.bearer(key), + json=SearchBody(query=query, max_results=2), + response_type=SearchResponse, + ) + ) + assert response.results and all(result.url for result in response.results), ( + f"/v1/search returned no results with a url: {response!r}" + ) + + generation: Final = _generation(client, langfuse_creds, alias=alias, started=started) + assert _input_text(generation) == query + output: Final = _output_text(generation) + assert output == "\n\n".join( + "\n".join(part for part in (result.title, result.url, result.snippet) if part) + for result in response.results + ), f"generation output is not the search results the caller received: {output!r}" diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index 8470d318db8..7366695c0d1 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -25,6 +25,8 @@ from e2e_http import ( unwrap, ) from models import ( + AuditLogPage, + AuditLogParams, ChatBody, ChatMessage, ConnectionTestBody, @@ -35,6 +37,7 @@ from models import ( CustomerResponse, KeyBlockBody, KeyDeleteBody, + KeyDeleteByAliasBody, KeyGenerateBody, KeyGenerateResponse, KeyInfoParams, @@ -159,6 +162,31 @@ class ManagementClient: def update_key_models(self, key: str, models: list[str]) -> None: _ = unwrap(self.update_key(KeyUpdateBody(key=key, models=models))) + def delete_key_by_alias(self, key_alias: str) -> None: + _ = unwrap( + self.proxy.transport.post( + "/key/delete", + headers=self.proxy.management_headers(), + json=KeyDeleteByAliasBody(key_aliases=[key_alias]), + response_type=NoBody, + ) + ) + + def key_deleted_audit_logs(self, token_hash: str) -> AuditLogPage: + return unwrap( + self.proxy.transport.get( + "/audit", + headers=self.proxy.management_headers(), + params=AuditLogParams( + object_id=token_hash, + action="deleted", + table_name="LiteLLM_VerificationToken", + page_size=100, + ), + response_type=AuditLogPage, + ) + ) + def key_info_as(self, key: str, *, caller_key: str | None = None) -> Result[KeyInfoResponse]: return self.proxy.transport.get( "/key/info", diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index 476165b715d..da0fc37aff8 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -16,8 +16,8 @@ from typing import Final import pytest -from e2e_config import UI_PASSWORD, UI_USERNAME, unique_marker -from e2e_http import StreamingResponse, Success +from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, UI_PASSWORD, UI_USERNAME, unique_marker +from e2e_http import StreamingResponse, Success, unwrap from lifecycle import ResourceManager from management_client import ( DASHBOARD_SESSION_TEAM_ID, @@ -26,7 +26,9 @@ from management_client import ( ManagementClient, ) from models import ( + AuditLogPage, KeyGenerateBody, + KeyGenerateResponse, KeyUpdateBody, LiteLLMParamsBody, ModelInfoEntry, @@ -40,6 +42,7 @@ from models import ( UserNewBody, UserUpdateBody, ) +from proxy_client import Converged, await_converged pytestmark = pytest.mark.e2e @@ -801,3 +804,128 @@ class TestCustomer: assert info.user_id == customer, ( f"/customer/info did not report the created end-user; got {info.user_id!r}" ) + + +def _await_deleted_audit_rows(client: ManagementClient, token_hash: str) -> AuditLogPage: + outcome = await_converged( + lambda: client.key_deleted_audit_logs(token_hash), + converged=lambda page: page.total >= 1, + timeout=POLL_TIMEOUT, + interval=POLL_INTERVAL, + now=time.monotonic, + sleep=time.sleep, + ) + return outcome.result if isinstance(outcome, Converged) else outcome.last_result + + +def _assert_single_deleted_row(page: AuditLogPage, token_hash: str) -> None: + assert page.total == 1, page + row = page.audit_logs[0] + assert row.action == "deleted", row + assert row.table_name == "LiteLLM_VerificationToken", row + assert row.object_id == token_hash, row + assert row.changed_by, row + + +def _assert_key_deleted(client: ManagementClient, key: str) -> None: + def gone() -> bool | None: + match client.key_info_as(key): + case Success(data=response): + return True if response.info.status == "deleted" else None + case _: + return True + + _ = _poll( + client, + gone, + "/key/info never reported status 'deleted' for a key whose deletion returned", + ) + + +def _token_of(created: KeyGenerateResponse) -> str: + assert created.token is not None, created + return created.token + + +def _generate_response( + client: ManagementClient, resources: ResourceManager, body: KeyGenerateBody +) -> KeyGenerateResponse: + created = unwrap(client.generate_key(body)) + resources.defer(lambda: client.delete_key_strict(created.key, missing_ok=True)) + return created + + +class TestKeyDeletionAuditLog: + @pytest.mark.covers("mgmt.key.delete.audit_logged") + def test_key_delete_by_key_writes_audit_row( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + created = _generate_response(client, resources, KeyGenerateBody(key_alias=f"e2e-audit-{unique_marker()}")) + token = _token_of(created) + + client.delete_key_strict(created.key) + + _assert_key_deleted(client, created.key) + _assert_single_deleted_row(_await_deleted_audit_rows(client, token), token) + + @pytest.mark.covers("mgmt.key.delete.audit_logged") + def test_key_delete_by_alias_writes_audit_row( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + alias = f"e2e-audit-{unique_marker()}" + created = _generate_response(client, resources, KeyGenerateBody(key_alias=alias)) + token = _token_of(created) + + client.delete_key_by_alias(alias) + + _assert_key_deleted(client, created.key) + _assert_single_deleted_row(_await_deleted_audit_rows(client, token), token) + + @pytest.mark.covers("mgmt.team.member_delete.audit_logs_keys") + def test_team_member_delete_writes_audit_row_for_member_keys( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + team_id = _create_team(client, resources, f"e2e-audit-team-{unique_marker()}", []) + user_id = _create_user( + client, + resources, + UserNewBody(user_email=f"e2e-audit-{unique_marker()}@example.com", user_role="internal_user"), + ) + client.add_team_member(team_id, user_id) + created = _generate_response(client, resources, KeyGenerateBody(user_id=user_id, team_id=team_id)) + token = _token_of(created) + + client.delete_team_member(team_id, user_id) + + _assert_key_deleted(client, created.key) + _assert_single_deleted_row(_await_deleted_audit_rows(client, token), token) + + @pytest.mark.covers("mgmt.team.delete.audit_logs_keys") + def test_team_delete_writes_audit_row_for_team_keys( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + team_id = _create_team(client, resources, f"e2e-audit-team-{unique_marker()}", []) + created = _generate_response(client, resources, KeyGenerateBody(team_id=team_id)) + token = _token_of(created) + + client.delete_team(team_id) + + _assert_key_deleted(client, created.key) + _assert_single_deleted_row(_await_deleted_audit_rows(client, token), token) + + @pytest.mark.covers("mgmt.user.delete.audit_logs_keys") + def test_user_delete_writes_audit_row_for_user_keys( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + user_id = _create_user( + client, + resources, + UserNewBody(user_email=f"e2e-audit-{unique_marker()}@example.com", user_role="internal_user"), + ) + created = _generate_response(client, resources, KeyGenerateBody(user_id=user_id)) + token = _token_of(created) + + client.delete_user_strict(user_id) + + _assert_key_deleted(client, created.key) + _assert_single_deleted_row(_await_deleted_audit_rows(client, token), token) diff --git a/tests/e2e/memory_readings.py b/tests/e2e/memory_readings.py new file mode 100644 index 00000000000..37a27c455f8 --- /dev/null +++ b/tests/e2e/memory_readings.py @@ -0,0 +1,78 @@ +"""Per-worker RSS readings of the proxy through /debug/memory/summary. + +One read goes to every configured replica (PROXY_REPLICA_URLS) under the master +key and answers from whichever worker behind that address took the connection; +the release stack runs one worker per gateway replica, so a read per replica is +a read per worker. A reading keys its worker by replica address, hostname, and +pid, since pods in their own pid namespaces report the same pids. A replica that +gives no reading (unreachable, a non-2xx, or a summary without ram_usage_mb) is +kept as a failure reason rather than dropped, so a test can fail on it by name +instead of passing on the replicas that did answer. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +from e2e_http import Result, Success +from models import MemorySummaryResponse +from proxy_client import ProxyClient + +WorkerKey = tuple[str, str | None, int] + + +@dataclass(frozen=True, slots=True) +class RssReading: + replica: str + hostname: str | None + worker_pid: int + ram_usage_mb: float + + @property + def worker(self) -> WorkerKey: + return (self.replica, self.hostname, self.worker_pid) + + @property + def where(self) -> str: + return f"worker pid {self.worker_pid} on {self.hostname or 'an unnamed host'} behind {self.replica}" + + +@dataclass(frozen=True, slots=True) +class RssCapture: + readings: tuple[RssReading, ...] + failures: tuple[str, ...] + + @property + def heaviest(self) -> RssReading | None: + return max(self.readings, key=lambda reading: reading.ram_usage_mb, default=None) + + @property + def junit_properties(self) -> tuple[tuple[str, object], ...]: + heaviest: Final = self.heaviest + if heaviest is None: + return () + return (("idle_rss_heaviest_mb", heaviest.ram_usage_mb), ("idle_rss_heaviest_worker", heaviest.where)) + + +def _outcome(replica: str, result: Result[MemorySummaryResponse]) -> RssReading | str: + match result: + case Success(data=body) if body.memory.ram_usage_mb is not None: + return RssReading(replica, body.hostname, body.worker_pid, body.memory.ram_usage_mb) + case Success(data=body): + return f"{replica} answered /debug/memory/summary without ram_usage_mb: {body.memory.error}" + case _: + return f"{replica} gave no /debug/memory/summary reading: {result}" + + +def rss_capture(summaries: Mapping[str, Result[MemorySummaryResponse]]) -> RssCapture: + outcomes: Final = tuple(_outcome(replica, result) for replica, result in summaries.items()) + return RssCapture( + readings=tuple(outcome for outcome in outcomes if isinstance(outcome, RssReading)), + failures=tuple(outcome for outcome in outcomes if isinstance(outcome, str)), + ) + + +def read_rss_everywhere(proxy: ProxyClient, *, timeout: float | None = None) -> RssCapture: + return rss_capture(proxy.memory_summary_everywhere(timeout=timeout)) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 2608da2e4fc..bc81d1015ba 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -124,6 +124,32 @@ class KeyDeleteBody(BaseModel): keys: list[str] +class KeyDeleteByAliasBody(BaseModel): + key_aliases: list[str] + + +class AuditLogParams(BaseModel): + object_id: str + action: str + table_name: str + page_size: int + + +class AuditLogEntry(BaseModel): + id: str + changed_by: str | None = None + changed_by_api_key: str | None = None + action: str + table_name: str + object_id: str + before_value: object | None = None + + +class AuditLogPage(BaseModel): + audit_logs: list[AuditLogEntry] + total: int + + class KeyInfoParams(BaseModel): key: str @@ -308,6 +334,7 @@ class ChatBody(BaseModel): tools: Sequence[ChatTool | McpChatTool] | None = None tool_choice: str | None = None guardrails: list[str] | None = None + include_guardrail_response: bool | None = None response_format: dict[str, object] | None = None chat_template_kwargs: dict[str, bool] | None = None cache: dict[str, bool] | None = {"no-cache": True} @@ -422,6 +449,14 @@ class Usage(BaseModel): completion_tokens_details: CompletionTokensDetails | None = None +class GuardrailInformationEntry(BaseModel): + guardrail_name: str + guardrail_status: str + guardrail_mode: object | None = None + guardrail_response: object | None = None + duration: float | None = None + + class ChatResponse(BaseModel): id: str | None = None object: str | None = None @@ -429,6 +464,7 @@ class ChatResponse(BaseModel): choices: list[ChatChoice] = [] usage: Usage | None = None service_tier: str | None = None + guardrail_information: list[GuardrailInformationEntry] | None = None # ---------- anthropic /v1/messages + count_tokens ---------- @@ -749,6 +785,12 @@ class OcrBody(BaseModel): document: OcrDocument +class OcrForm(BaseModel): + """Multipart /v1/ocr form fields; the document travels as the `file` part.""" + + model: str + + class OcrPage(BaseModel): index: int markdown: str @@ -760,6 +802,122 @@ class OcrResponse(BaseModel): pages: list[OcrPage] = [] +# ---------- completions ---------- + + +class CompletionBody(BaseModel): + model: str + prompt: str + max_tokens: int = 8 + n: int = 1 + + +class CompletionChoice(BaseModel): + text: str = "" + + +class CompletionResponse(BaseModel): + choices: list[CompletionChoice] = [] + + +# ---------- images ---------- + + +class ImageGenerationBody(BaseModel): + model: str + prompt: str + n: int = 1 + size: str = "1024x1024" + quality: str = "low" + + +class ImageDatum(BaseModel): + url: str | None = None + b64_json: str | None = None + + +class ImageGenerationResponse(BaseModel): + data: list[ImageDatum] = [] + + +class ImageEditForm(BaseModel): + """POST /v1/images/edits form fields; the image travels as the `image` multipart part.""" + + model: str + prompt: str + size: str = "1024x1024" + quality: str = "low" + + +class SearchBody(BaseModel): + """POST /v1/search/{search_tool_name} body (Perplexity-compatible).""" + + query: str + max_results: int = 2 + + +class SearchResultItem(BaseModel): + title: str = "" + url: str = "" + snippet: str = "" + + +class SearchResponse(BaseModel): + results: list[SearchResultItem] = [] + + +class SearchToolLiteLLMParamsBody(BaseModel): + search_provider: str + + +class SearchToolBody(BaseModel): + search_tool_name: str + litellm_params: SearchToolLiteLLMParamsBody + + +class SearchToolCreateBody(BaseModel): + """POST /search_tools body: the tool as it would sit under `search_tools:` in the config.""" + + search_tool: SearchToolBody + + +class SearchToolCreateResponse(BaseModel): + search_tool_id: str + + +# ---------- audio ---------- + + +class SpeechBody(BaseModel): + model: str + input: str + voice: str = "alloy" + + +class TranscriptionForm(BaseModel): + model: str + + +class TranscriptionResponse(BaseModel): + text: str = "" + + +# ---------- moderations ---------- + + +class ModerationBody(BaseModel): + model: str + input: str + + +class ModerationResult(BaseModel): + flagged: bool + + +class ModerationResponse(BaseModel): + results: list[ModerationResult] = [] + + # ---------- spend logs ---------- @@ -794,6 +952,7 @@ class SpendLogRow(BaseModel): cache_hit: str | None = None call_type: str | None = None custom_llm_provider: str | None = None + model_id: str | None = None team_id: str | None = None user: str | None = None end_user: str | None = None @@ -801,8 +960,11 @@ class SpendLogRow(BaseModel): completion_tokens: int | None = None total_tokens: int | None = None request_tags: list[str] | None = None + session_id: str | None = None metadata: SpendLogMetadata | None = None proxy_server_request: JsonValue = None + response: JsonValue = None + litellm_call_id: str | None = None class SpendLogs(RootModel[list[SpendLogRow]]): @@ -835,6 +997,15 @@ class SpendLogsPageParams(BaseModel): api_key: str | None = None +class SessionSpendLogsParams(BaseModel): + """Query for /spend/logs/session/ui, the session view the Admin UI logs page + opens: every row whose session_id equals the given one, newest first.""" + + session_id: str + page: int = 1 + page_size: int = 100 + + class SpendLogsPage(BaseModel): data: list[SpendLogRow] = [] total: int diff --git a/tests/e2e/other/other_client.py b/tests/e2e/other/other_client.py index 4313bbe4068..057a735e5e5 100644 --- a/tests/e2e/other/other_client.py +++ b/tests/e2e/other/other_client.py @@ -15,15 +15,25 @@ from __future__ import annotations from dataclasses import dataclass -from e2e_http import NoBody, ProbeResult, Result +from e2e_http import AuthHeaders, NoBody, ProbeResult, Result from idp import Keycloak, keycloak_from_env from models import ( + ChatBody, + ChatResponse, ReadinessDetailsResponse, ReadinessResponse, UserListParams, UserListResponse, ) from proxy_client import ProxyClient +from pydantic import Field + + +class TeamHeaders(AuthHeaders): + """Bearer auth plus ``x-litellm-team-id``, the header a JWT caller sends to + pick one of the teams it belongs to.""" + + x_litellm_team_id: str = Field(serialization_alias="x-litellm-team-id") @dataclass(frozen=True, slots=True) @@ -66,6 +76,18 @@ class OtherClient: response_type=ReadinessDetailsResponse, ) + def chat_as_team(self, token: str, team: str, body: ChatBody) -> Result[ChatResponse]: + """POST /chat/completions under `token` with `x-litellm-team-id: team`.""" + return self.proxy.transport.post( + "/chat/completions", + headers=TeamHeaders( + authorization=self.proxy.transport.bearer(token).authorization, + x_litellm_team_id=team, + ), + json=body, + response_type=ChatResponse, + ) + def list_users_as(self, key: str) -> Result[UserListResponse]: """GET /user/list under `key`. Admin-only, so it doubles as the master key's authorization proof: the master key (proxy admin) reads it, a diff --git a/tests/e2e/other/test_jwt_auth_e2e.py b/tests/e2e/other/test_jwt_auth_e2e.py index 2bed40f4d69..de06c376f76 100644 --- a/tests/e2e/other/test_jwt_auth_e2e.py +++ b/tests/e2e/other/test_jwt_auth_e2e.py @@ -4,6 +4,7 @@ from __future__ import annotations import base64 import time +from dataclasses import dataclass from typing import Final import pytest @@ -52,6 +53,31 @@ def identity(client: OtherClient, resources: ResourceManager) -> Identity: return provisioned +@dataclass(frozen=True, slots=True) +class BoundTeam: + identity: Identity + team_id: str + team_alias: str + + +def _team(client: OtherClient, resources: ResourceManager, *, marker: str, team_id: str) -> str: + """A litellm team whose alias differs from its id, so a header naming one + cannot accidentally match the other.""" + team_alias: Final = f"e2e-jwt-alias-{marker}" + created: Final = client.proxy.create_team(TeamNewBody(team_alias=team_alias, team_id=team_id)) + resources.defer(lambda: client.proxy.delete_team(created)) + return team_alias + + +@pytest.fixture +def bound_team(client: OtherClient, resources: ResourceManager) -> BoundTeam: + """An identity whose single group is a real team, plus that team's alias.""" + marker: Final = unique_marker() + provisioned: Final = _provision(client, resources, marker=marker) + team_alias: Final = _team(client, resources, marker=marker, team_id=provisioned.group) + return BoundTeam(identity=provisioned, team_id=provisioned.group, team_alias=team_alias) + + def _ping() -> ChatBody: return ChatBody( model=CHEAP_OPENAI_MODEL, @@ -155,3 +181,58 @@ class TestJwtAuth: def test_plain_virtual_key_still_works_with_jwt_auth_enabled(self, client: OtherClient, scoped_key: str) -> None: response: Final = unwrap(client.proxy.chat(scoped_key, _ping())) assert response.choices, f"an sk- key must keep working on a proxy with enable_jwt_auth, got {response}" + + +def _team_of_request(client: OtherClient, token: str, team: str) -> str | None: + response: Final = unwrap(client.chat_as_team(token, team, _ping())) + assert response.id is not None and response.choices, ( + f"chat with x-litellm-team-id={team!r} returned no completion: {response}" + ) + rows: Final = client.proxy.poll_logs_for_request_id(response.id) + assert rows, f"no spend log row for request {response.id} within the poll deadline" + return rows[0].team_id + + +def _denial(client: OtherClient, token: str, team: str) -> str: + result: Final = client.chat_as_team(token, team, _ping()) + assert isinstance(result, UnknownApiError) and result.status_code == 403, ( + f"x-litellm-team-id={team!r} names no team the caller is in, so it must be rejected with 403, got {result}" + ) + assert team in result.body, f"the 403 must name the header value it rejected ({team!r}), got {result.body[:300]}" + return result.body + + +class TestJwtTeamHeader: + @pytest.mark.covers("other.auth.jwt.team_header_alias_binds_team") + def test_team_header_with_the_team_alias_binds_the_same_team_as_the_team_id( + self, client: OtherClient, bound_team: BoundTeam + ) -> None: + token: Final = client.idp.access_token(bound_team.identity) + assert bound_team.team_alias != bound_team.team_id + + by_id: Final = _team_of_request(client, token, bound_team.team_id) + assert by_id == bound_team.team_id, ( + f"precondition: x-litellm-team-id with the team id must bind {bound_team.team_id!r}, got {by_id!r}" + ) + + by_alias: Final = _team_of_request(client, token, bound_team.team_alias) + assert by_alias == bound_team.team_id, ( + f"x-litellm-team-id={bound_team.team_alias!r} must bind the same team as its id " + f"{bound_team.team_id!r}, got {by_alias!r}" + ) + + @pytest.mark.covers("other.auth.jwt.team_header_non_member_alias_denied") + def test_team_header_with_the_alias_of_a_team_the_caller_is_not_in_is_rejected_like_an_unknown_value( + self, client: OtherClient, resources: ResourceManager, bound_team: BoundTeam + ) -> None: + token: Final = client.idp.access_token(bound_team.identity) + other_marker: Final = unique_marker() + other_alias: Final = _team(client, resources, marker=other_marker, team_id=f"e2e-jwt-other-{other_marker}") + unknown: Final = f"e2e-jwt-unknown-{unique_marker()}" + + for_other_alias: Final = _denial(client, token, other_alias) + for_unknown: Final = _denial(client, token, unknown) + assert for_other_alias.replace(other_alias, "") == for_unknown.replace(unknown, ""), ( + "a non-member alias and an unknown value must get the same denial body, so the response does not " + f"reveal whether the team exists; got {for_other_alias[:300]!r} vs {for_unknown[:300]!r}" + ) diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 76c57452c69..51ea9fbe7bd 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -85,6 +85,9 @@ from models import ( RerankResponse, RouterCurrentValues, RouterSettingsResponse, + SearchToolCreateBody, + SearchToolCreateResponse, + SessionSpendLogsParams, SpendLogRow, SpendLogs, SpendLogsPage, @@ -505,13 +508,16 @@ class ProxyClient: ) ).info - def memory_summary_everywhere(self) -> Mapping[str, Result[MemorySummaryResponse]]: + def memory_summary_everywhere( + self, *, timeout: float | None = None + ) -> Mapping[str, Result[MemorySummaryResponse]]: return { url: transport.get( "/debug/memory/summary", headers=self.management_headers(transport=transport), params=NoBody(), response_type=MemorySummaryResponse, + timeout=timeout, ) for url, transport in self.replicas.items() } @@ -859,6 +865,30 @@ class ProxyClient: response_type=NoBody, ) + def create_search_tool(self, body: SearchToolCreateBody) -> str: + """POST /search_tools: register a search tool on the running proxy and return its id + once every worker has had a config-reload window to pick it up from the DB.""" + search_tool_id: Final = unwrap( + self.transport.post( + "/search_tools", + headers=self.management_headers(), + json=body, + response_type=SearchToolCreateResponse, + ) + ).search_tool_id + settle_propagation(time.monotonic()) + return search_tool_id + + def delete_search_tool(self, search_tool_id: str) -> None: + result = self.transport.delete( + f"/search_tools/{search_tool_id}", + headers=self.management_headers(), + json=NoBody(), + response_type=NoBody, + ) + if not is_ok(result): + warnings.warn(f"delete_search_tool({search_tool_id!r}) failed: {result}", stacklevel=2) + def create_credential(self, body: CredentialCreateBody) -> None: unwrap( self.transport.post( @@ -978,19 +1008,26 @@ class ProxyClient: response_type=CountTokensResponse, ) - def messages(self, key: str, body: AnthropicMessagesBody) -> Result[AnthropicMessagesResponse]: + def messages( + self, key: str, body: AnthropicMessagesBody, *, session_id: str | None = None + ) -> Result[AnthropicMessagesResponse]: """POST /v1/messages (Anthropic-native). The response is either the Anthropic-shape passthrough (`content`) or the OpenAI-normalized shape - (`choices`); AnthropicMessagesResponse models both.""" + (`choices`); AnthropicMessagesResponse models both. `session_id` goes out + as the `x-litellm-session-id` header, the way Claude Code sends it through + ANTHROPIC_CUSTOM_HEADERS, so every spend row the call produces shares it.""" return self.transport.post( "/v1/messages", - headers=self._anthropic_headers(key), + headers=self._anthropic_headers(key, session_id=session_id), json=body, response_type=AnthropicMessagesResponse, ) - def _anthropic_headers(self, key: str) -> AnthropicHeaders: - return AnthropicHeaders(authorization=self.transport.bearer(key).authorization) + def _anthropic_headers(self, key: str, *, session_id: str | None = None) -> AnthropicHeaders: + return AnthropicHeaders( + authorization=self.transport.bearer(key).authorization, + x_litellm_session_id=session_id, + ) # ---- spend read-back ------------------------------------------------ @@ -1034,6 +1071,27 @@ class ProxyClient: ) -> list[SpendLogRow]: return self._poll(lambda: self.spend_logs(SpendLogsParams(api_key=key)), min_rows, predicate) + def session_spend_logs(self, session_id: str) -> list[SpendLogRow]: + """GET /spend/logs/session/ui, the per-session view the Admin UI logs page + opens when a session id is clicked.""" + return unwrap( + self.transport.get( + "/spend/logs/session/ui", + headers=self.management_headers(), + params=SessionSpendLogsParams(session_id=session_id), + response_type=SpendLogsPage, + ) + ).data + + def poll_logs_for_session( + self, + session_id: str, + *, + min_rows: int = 1, + predicate: RowsPredicate | None = None, + ) -> list[SpendLogRow]: + return self._poll(lambda: self.session_spend_logs(session_id), min_rows, predicate) + def poll_logs_for_request_id( self, request_id: str, diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index c6acd449884..4866511af3f 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -12,5 +12,8 @@ markers = prompt_caching_stack: needs a proxy running with router_settings.optional_pre_call_checks including prompt_caching; deselected unless E2E_PROMPT_CACHING_STACK is set cli_determinism: drives the real claude CLI for several seconds; deselected unless E2E_CLI_DETERMINISM is set redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set + quiet_stack: measures the proxy itself, so it runs while no other test on this host is hitting the stack; every other test waits for it to finish mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless E2E_MCP_OAUTH_LIVE is set provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set + otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set + otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set diff --git a/tests/e2e/quota_management/ratelimit/test_model_group_alias_rate_limit_e2e.py b/tests/e2e/quota_management/ratelimit/test_model_group_alias_rate_limit_e2e.py new file mode 100644 index 00000000000..1c3fff47b78 --- /dev/null +++ b/tests/e2e/quota_management/ratelimit/test_model_group_alias_rate_limit_e2e.py @@ -0,0 +1,85 @@ +"""Live e2e: a model group alias must share its per-key deployment rate-limit +bucket with the model group it resolves to. + +Covers quota_management.ratelimit.model_group_alias.shares_bucket: the proxy +config declares `e2e-alias-rl-target` (a cheap Anthropic deployment) with +`default_api_key_rpm_limit: 3` and `router_settings.model_group_alias` mapping +`e2e-alias-rl-alias` -> `e2e-alias-rl-target`. Both spellings must draw on +one per-key rpm bucket, so a key that exhausts the limit on one spelling is +blocked on the other spelling inside the same window; each test in this file +exhausts the budget on one name and asserts the other name 429s. + +All calls of one test must land inside a single window +(LITELLM_RATE_LIMIT_WINDOW_SIZE, 60s default), which real chat latency +comfortably allows. +""" + +from __future__ import annotations + +import time + +import pytest +from e2e_config import unique_marker +from e2e_http import StreamingResponse, require_successful_call +from quota_client import QuotaClient + +pytestmark = pytest.mark.e2e + +MODEL_GROUP = "e2e-alias-rl-target" +MODEL_ALIAS = "e2e-alias-rl-alias" +RPM_LIMIT = 3 +WINDOW_SECONDS = 60 +LAST_CALL_LATENCY_MARGIN_SECONDS = 10 + + +def _chat(client: QuotaClient, key: str, model: str) -> StreamingResponse: + return client.chat(key, model, f"reply with one word {unique_marker()}") + + +def _exhaust_rpm(client: QuotaClient, key: str, model: str) -> float: + """Send RPM_LIMIT successful calls on `model`, opening the rate-limit + window; returns the send timestamp of the first call as a lower bound on + the window start. A fresh key may briefly 401 until the data plane's auth + cache picks it up, so retry on 401 to a deadline; a 401 never reaches the + rate limiter.""" + deadline = time.monotonic() + client.proxy.poll_timeout + first_sent_at: float | None = None + sent = 0 + while sent < RPM_LIMIT: + if first_sent_at is None: + first_sent_at = time.monotonic() + outcome = _chat(client, key, model) + if outcome.status_code == 401 and time.monotonic() < deadline: + time.sleep(client.proxy.poll_interval) + continue + require_successful_call(outcome) + sent += 1 + assert first_sent_at is not None + return first_sent_at + + +def _assert_blocked_inside_window( + client: QuotaClient, key: str, model: str, window_opened_at: float +) -> StreamingResponse: + assert time.monotonic() < window_opened_at + WINDOW_SECONDS - LAST_CALL_LATENCY_MARGIN_SECONDS, ( + f"the {RPM_LIMIT} exhaust calls took too long; the follow-up call could land in the " + "next window and mask a shared-bucket regression" + ) + outcome = _chat(client, key, model) + assert outcome.status_code == 429, ( + f"{model} must share its rpm bucket with the model group/alias that was already " + f"exhausted, expected a 429 but got {outcome.status_code}: {outcome.body[:300]}" + ) + return outcome + + +class TestModelGroupAliasRateLimit: + @pytest.mark.covers("quota_management.ratelimit.model_group_alias.shares_bucket") + def test_alias_shares_rpm_bucket_with_model_group(self, client: QuotaClient, scoped_key: str) -> None: + opened_at = _exhaust_rpm(client, scoped_key, MODEL_GROUP) + _assert_blocked_inside_window(client, scoped_key, MODEL_ALIAS, opened_at) + + @pytest.mark.covers("quota_management.ratelimit.model_group_alias.shares_bucket") + def test_model_group_shares_rpm_bucket_with_alias(self, client: QuotaClient, scoped_key: str) -> None: + opened_at = _exhaust_rpm(client, scoped_key, MODEL_ALIAS) + _assert_blocked_inside_window(client, scoped_key, MODEL_GROUP, opened_at) diff --git a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py index b7f59fe5f89..8b63b063e14 100644 --- a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py +++ b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py @@ -12,9 +12,10 @@ helpers from one place. from __future__ import annotations import time -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from types import MappingProxyType from typing import Final from e2e_config import unique_marker @@ -48,6 +49,7 @@ from models import ( SpendLogsPageParams, SpendTagsResponse, TagSpend, + TeamInfoParams, UserDeleteBody, UserDeleteResponse, UserNewBody, @@ -57,6 +59,8 @@ from models import ( from proxy_client import Converged, ProxyClient, await_converged from pydantic import BaseModel, Field +METRICS_PATH: Final = "/metrics/" + __all__ = [ "BatchCreateBody", "CallbackLogMetadata", @@ -189,6 +193,7 @@ class DailyActivityKeyMetadata(BaseModel): class DailyActivityKeyMetrics(BaseModel): api_requests: int = 0 + spend: float = 0.0 class DailyActivityKeyBreakdown(BaseModel): @@ -209,6 +214,14 @@ class DailyActivityResponse(BaseModel): results: list[DailyActivityRow] = [] +class TeamInfoSpend(BaseModel): + spend: float | None = None + + +class TeamInfoSpendResponse(BaseModel): + team_info: TeamInfoSpend + + def _chat_body( model: str, content: str, @@ -334,6 +347,42 @@ class SpendClient: time.sleep(self.proxy.poll_interval) return spend + def team_spend(self, team_id: str) -> float: + return ( + unwrap( + self.proxy.transport.get( + "/team/info", + headers=self.proxy.transport.master, + params=TeamInfoParams(team_id=team_id), + response_type=TeamInfoSpendResponse, + ) + ).team_info.spend + or 0.0 + ) + + def poll_team_spend(self, team_id: str, *, minimum: float = 0.0) -> float: + outcome: Final = await_converged( + lambda: self.team_spend(team_id), + converged=lambda spend: spend > minimum, + timeout=self.proxy.poll_timeout, + interval=self.proxy.poll_interval, + now=time.monotonic, + sleep=time.sleep, + ) + return outcome.result if isinstance(outcome, Converged) else outcome.last_result + + def scrape_metrics(self) -> Mapping[str, ProbeResult]: + """GET /metrics/ on every replica in PROXY_REPLICA_URLS, keyed by replica. The + counter is per pod, so the union of the replicas is the fleet's exposition; the + trailing slash is the mounted app's own path, since bare /metrics answers a 307 + whose Location drops the port behind a Host-rewriting balancer.""" + return MappingProxyType( + { + replica: transport.probe(METRICS_PATH, params=NoBody()) + for replica, transport in self.proxy.replicas.items() + } + ) + def spend_logs_page( self, *, api_key: str | None, page: int, page_size: int ) -> SpendLogsPage: @@ -500,9 +549,21 @@ class SpendClient: return self.proxy.transport.probe("/health", params=HealthParams(model=model)) def daily_activity_for_key(self, token: str, *, start: datetime, end: datetime) -> DailyActivityKeyBreakdown | None: + return self._key_breakdown("/user/daily/activity", token, start=start, end=end) + + def usage_export_row_for_key( + self, token: str, *, start: datetime, end: datetime + ) -> DailyActivityKeyBreakdown | None: + """The key's row on /user/daily/activity/aggregated, the response the + dashboard's Export Usage Data CSV serializes.""" + return self._key_breakdown("/user/daily/activity/aggregated", token, start=start, end=end) + + def _key_breakdown( + self, route: str, token: str, *, start: datetime, end: datetime + ) -> DailyActivityKeyBreakdown | None: response: Final = unwrap( self.proxy.transport.get( - "/user/daily/activity", + route, headers=self.proxy.transport.master, params=DailyActivityParams( start_date=start.strftime("%Y-%m-%d"), @@ -519,9 +580,21 @@ class SpendClient: def poll_daily_activity_for_key( self, token: str, *, start: datetime, end: datetime, min_requests: int + ) -> DailyActivityKeyBreakdown | None: + return self._poll_key_breakdown(lambda: self.daily_activity_for_key(token, start=start, end=end), min_requests) + + def poll_usage_export_row_for_key( + self, token: str, *, start: datetime, end: datetime, min_requests: int + ) -> DailyActivityKeyBreakdown | None: + return self._poll_key_breakdown( + lambda: self.usage_export_row_for_key(token, start=start, end=end), min_requests + ) + + def _poll_key_breakdown( + self, fetch: Callable[[], DailyActivityKeyBreakdown | None], min_requests: int ) -> DailyActivityKeyBreakdown | None: outcome: Final = await_converged( - lambda: self.daily_activity_for_key(token, start=start, end=end), + fetch, converged=lambda found: found is not None and found.metrics.api_requests >= min_requests, timeout=self.proxy.poll_timeout, interval=self.proxy.poll_interval, diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_surface_consistency_e2e.py b/tests/e2e/quota_management/spend_tracking/test_spend_surface_consistency_e2e.py new file mode 100644 index 00000000000..9033b9d75c4 --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/test_spend_surface_consistency_e2e.py @@ -0,0 +1,161 @@ +"""One priced request must land the same response_cost on every spend surface. + +A customer reconciles the bill from whichever surface they look at: the spend +log row, the key's and the team's rolled-up spend on /key/info and /team/info, +the usage page's Export Usage Data CSV (the dashboard serializes the +/user/daily/activity/aggregated rows it already holds; there is no server-side +CSV endpoint), and the litellm_spend_metric counter Prometheus scrapes. Each is +written by a different writer (the spend log insert, the key and team rollups in +db_spend_update_writer, the daily spend tables, the Prometheus success callback), +so one of them can drift without the others noticing: the cause of the +key-versus-log mismatch in LIT-3620 and the export-versus-console mismatch in +LIT-5045. The deployment carries its own per-token rates, so the expected cost +is computed from the returned usage rather than read off any one surface, and +every surface is held to that number. + +/metrics is per pod, so every replica the stack exports (PROXY_REPLICA_URLS) is +scraped directly and the samples merged; a stack that exports only its balancer +is scraped there until the pod that served the call answers. The request itself +is sent once. +""" + +from __future__ import annotations + +import time +from collections.abc import Mapping +from datetime import datetime, timedelta, timezone +from itertools import groupby +from math import isclose +from types import MappingProxyType +from typing import Final + +import pytest +from e2e_config import provider_edge_base, unique_marker +from e2e_http import ProbeResult +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, KeyGenerateBody, LiteLLMParamsBody, TeamNewBody +from prometheus_client.parser import text_string_to_metric_families +from proxy_client import Converged, await_converged +from spend_e2e_client import SpendClient, unwrap +from spend_reconciliation import INPUT_RATE, OUTPUT_RATE + +pytestmark = pytest.mark.e2e + +SPEND_METRIC: Final = "litellm_spend_metric_total" +KEY_HASH_LABEL: Final = "hashed_api_key" +TEAM_LABEL: Final = "team" + +SeriesLabels = tuple[tuple[str, str], ...] + + +def _spend_series_for_key(scrapes: Mapping[str, ProbeResult], token: str) -> Mapping[SeriesLabels, float]: + samples: Final = sorted( + (tuple(sorted(sample.labels.items())), sample.value) + for scrape in scrapes.values() + if scrape.status_code == 200 + for family in text_string_to_metric_families(scrape.body) + for sample in family.samples + if sample.name == SPEND_METRIC and sample.labels.get(KEY_HASH_LABEL) == token + ) + return MappingProxyType( + {labels: sum(value for _, value in group) for labels, group in groupby(samples, key=lambda sample: sample[0])} + ) + + +def _poll_spend_series_for_key(client: SpendClient, token: str) -> Mapping[SeriesLabels, float]: + outcome: Final = await_converged( + client.scrape_metrics, + converged=lambda scrapes: bool(_spend_series_for_key(scrapes, token)), + timeout=client.proxy.poll_timeout, + interval=client.proxy.poll_interval, + now=time.monotonic, + sleep=time.sleep, + ) + scrapes: Final = outcome.result if isinstance(outcome, Converged) else outcome.last_result + assert _spend_series_for_key(scrapes, token), ( + f"{SPEND_METRIC} never exposed a series for {KEY_HASH_LABEL}={token} on any replica; " + f"last scrape status per replica: {({replica: scrape.status_code for replica, scrape in scrapes.items()})}" + ) + return _spend_series_for_key(scrapes, token) + + +def _same_spend(actual: float | None, expected: float) -> bool: + return actual is not None and isclose(actual, expected, rel_tol=1e-6, abs_tol=1e-9) + + +class TestSpendSurfaceConsistency: + @pytest.mark.replayable + @pytest.mark.covers("quota_management.spend_tracking.surface_consistency.matches_every_surface") + def test_one_request_lands_the_same_spend_on_every_surface( + self, client: SpendClient, resources: ResourceManager + ) -> None: + started: Final = datetime.now(timezone.utc) + marker: Final = unique_marker() + base: Final = provider_edge_base("openai") + model: Final = f"e2e-spend-surfaces-{marker}" + model_id: Final = client.proxy.create_model( + model, + LiteLLMParamsBody( + model="openai/gpt-5.6-luna", + api_key="os.environ/OPENAI_API_KEY", + api_base=None if base is None else f"{base}/v1", + input_cost_per_token=INPUT_RATE, + output_cost_per_token=OUTPUT_RATE, + ), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + team_id: Final = client.proxy.create_team(TeamNewBody(team_alias=f"e2e-spend-surfaces-{marker}")) + resources.defer(lambda: client.proxy.delete_team(team_id)) + record: Final = client.generate_key_record( + KeyGenerateBody(team_id=team_id, models=[model], key_alias=f"e2e-spend-surfaces-{marker}") + ) + resources.defer(lambda: client.proxy.delete_key(record.key)) + assert record.token, "/key/generate answered without the key's token hash" + token: Final = record.token + + response: Final = unwrap( + client.proxy.chat( + record.key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content=f"Reply with one word. {marker}")], + max_completion_tokens=128, + ), + ) + ) + usage: Final = response.usage + assert response.id, "successful response must have an ID" + assert usage is not None and usage.prompt_tokens and usage.completion_tokens, f"no billable usage: {usage}" + expected: Final = usage.prompt_tokens * INPUT_RATE + usage.completion_tokens * OUTPUT_RATE + + rows: Final = client.proxy.poll_logs_for_request_id(response.id) + assert len(rows) == 1, f"expected one spend row for {response.id}, saw {len(rows)}: {rows}" + row: Final = rows[0] + assert row.api_key == token, f"spend row keyed by {row.api_key}, not the key's token hash {token}" + assert row.team_id == team_id, f"spend row attributed to team {row.team_id}, not {team_id}" + assert row.status == "success", f"spend row status {row.status}" + + key_spend: Final = client.poll_key_spend(record.key, minimum=expected * 0.999999) + team_spend: Final = client.poll_team_spend(team_id, minimum=expected * 0.999999) + export_row: Final = client.poll_usage_export_row_for_key( + token, start=started - timedelta(days=1), end=datetime.now(timezone.utc), min_requests=1 + ) + assert export_row is not None, f"/user/daily/activity/aggregated never listed key {token} under api_keys" + series: Final = _poll_spend_series_for_key(client, token) + off_team: Final = tuple(labels for labels in series if dict(labels).get(TEAM_LABEL) != team_id) + assert not off_team, f"{SPEND_METRIC} series for the key carry a team other than {team_id}: {off_team}" + + observed: Final = MappingProxyType( + { + "/spend/logs row": row.spend, + "/key/info spend": key_spend, + "/team/info spend": team_spend, + "usage export row (/user/daily/activity/aggregated)": export_row.metrics.spend, + SPEND_METRIC: sum(series.values()), + } + ) + drifted: Final = tuple(surface for surface, spend in observed.items() if not _same_spend(spend, expected)) + assert not drifted, ( + f"response_cost {expected} (usage {usage.prompt_tokens}x{INPUT_RATE} + " + f"{usage.completion_tokens}x{OUTPUT_RATE}) drifted on {drifted}; every surface: {dict(observed)}" + ) diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py index 8a91e53e7d7..6ca6f8cee55 100644 --- a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py @@ -21,9 +21,9 @@ from math import isclose from typing import Final import pytest -from e2e_http import Success +from e2e_http import RateLimitedError, Success from lifecycle import ResourceManager -from models import LiteLLMParamsBody, SpendLogs, SpendLogsParams +from models import KeyGenerateBody, LiteLLMParamsBody, SpendLogs, SpendLogsParams from spend_e2e_client import SpendClient, SpendLogRow, is_ok, unique_marker, unwrap pytestmark = pytest.mark.e2e @@ -43,6 +43,7 @@ def _summarize(rows: list[SpendLogRow]) -> list[dict[str, object]]: "cache_hit", "call_type", "custom_llm_provider", + "model_id", "prompt_tokens", "completion_tokens", "total_tokens", @@ -498,6 +499,47 @@ def test_failure_call_writes_failure_status_row( assert (failure_row.spend or 0) == 0.0, "failed call must not be charged" +@pytest.mark.covers("quota_management.spend_tracking.failure.attributes_provider") +def test_pre_call_rejection_row_attributes_provider_and_model_id( + client: SpendClient, resources: ResourceManager +) -> None: + """A request the proxy rejects before the router picks a deployment (here the + key's rpm limit, a pre_call_hook 429) never reaches the code that stamps the + deployment onto the log. The failure row must still carry the provider and + model_id of the model group's only deployment, so per-provider failure reports + can count it.""" + model = f"e2e-spend-precall-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model="openai/gpt-5.5", api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = client.proxy.generate_key(KeyGenerateBody(models=[model], rpm_limit=1)) + resources.defer(lambda: client.proxy.delete_key(key)) + + unwrap(client.chat(key, model, f"reply with one word {unique_marker()}", max_tokens=8)) + rejected = client.chat(key, model, f"over the rpm limit {unique_marker()}", max_tokens=8) + assert isinstance(rejected, RateLimitedError), ( + f"the second call on an rpm_limit=1 key must be rejected with 429 before routing, got {rejected}" + ) + + rows = client.poll_logs_for_key( + key, + min_rows=2, + predicate=lambda rs: {r.status for r in rs} >= {"success", "failure"}, + ) + success_row = _require_row(rows, lambda r: r.status == "success", "for the served call") + failure_row = _require_row(rows, lambda r: r.status == "failure", "for the rate-limited call") + + assert failure_row.custom_llm_provider == success_row.custom_llm_provider, ( + f"rejected call lost its provider: failure row {failure_row.custom_llm_provider!r} vs " + f"served row {success_row.custom_llm_provider!r}; {_summarize(rows)}" + ) + assert failure_row.model_id == model_id, ( + f"rejected call lost its deployment: failure row model_id {failure_row.model_id!r} vs " + f"registered {model_id!r}; {_summarize(rows)}" + ) + + @pytest.mark.covers("quota_management.spend_tracking.spend_calculate.returns_cost") def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None: cost = client.calculate_spend( diff --git a/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py b/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py new file mode 100644 index 00000000000..89d0beec414 --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/test_websearch_interception_session_e2e.py @@ -0,0 +1,102 @@ +"""Intercepted web searches are billed under the LLM request's session. + +The websearch_interception callback turns an Anthropic ``web_search`` server tool +into a ``litellm.asearch`` call against a configured search tool, so each search is +its own spend row (call_type ``asearch``) next to the ``anthropic_messages`` row for +the turn that asked for it. Claude Code and the Admin UI group spend by +``session_id``, so the search row has to carry the same session as the turn that +triggered it; before the fix it landed under a session of its own and the session +view under-counted both requests and spend (LIT-8063). + +Needs a proxy booted with the callback and a real search backend, which +``gateway/stage_mirror_ci_config.yml`` carries as the ``e2e-search`` Perplexity tool. +""" + +from typing import Final + +import pytest +from e2e_config import unique_marker +from e2e_http import unwrap +from lifecycle import ResourceManager +from models import ( + AnthropicMessagesBody, + AnthropicWebSearchTool, + ChatMessage, + LiteLLMParamsBody, + SpendLogRow, +) +from proxy_client import ProxyClient + +pytestmark = pytest.mark.e2e + +BEDROCK_INVOKE_BACKEND: Final = "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0" +SEARCH_CALL_TYPE: Final = "asearch" + + +def _has_search_row(rows: list[SpendLogRow]) -> bool: + return any(row.call_type == SEARCH_CALL_TYPE for row in rows) + + +class TestWebSearchInterceptionSession: + @pytest.mark.covers( + "quota_management.spend_tracking.websearch_interception.bills_under_request_session", + exercised_on=("messages",), + ) + def test_intercepted_search_is_billed_under_the_request_session( + self, proxy: ProxyClient, resources: ResourceManager + ) -> None: + """One /v1/messages turn that runs an intercepted web search must produce an + ``asearch`` spend row in the same session as its ``anthropic_messages`` row, + billed separately and with its own request id.""" + marker: Final = unique_marker() + model: Final = f"e2e-websearch-session-{marker}" + model_id: Final = proxy.create_model( + model, LiteLLMParamsBody(model=BEDROCK_INVOKE_BACKEND, aws_region_name="us-east-1") + ) + resources.defer(lambda: proxy.delete_model(model_id)) + key: Final = resources.key(models=[model]) + session_id: Final = f"e2e-websearch-session-{marker}" + + response: Final = unwrap( + proxy.messages( + key, + AnthropicMessagesBody( + model=model, + max_tokens=512, + tools=[AnthropicWebSearchTool(type="web_search_20250305", name="web_search", max_uses=1)], + messages=[ + ChatMessage( + role="user", + content=f"Use web search to find one recent news headline about Anthropic ({marker}).", + ) + ], + ), + session_id=session_id, + ) + ) + block_types: Final = tuple(block.type for block in response.content or ()) + assert "web_search_tool_result" in block_types, ( + f"precondition: the turn never ran an intercepted search, so there is no search row to attribute. " + f"blocks={block_types}" + ) + + rows: Final = proxy.poll_logs_for_session(session_id, min_rows=2, predicate=_has_search_row) + by_call_type: Final = {row.call_type or "" for row in rows} + assert SEARCH_CALL_TYPE in by_call_type, ( + f"session {session_id} has no {SEARCH_CALL_TYPE} row, so the intercepted search was billed under a " + f"different session and the session view misses its cost. call_types={sorted(by_call_type)} " + f"rows={[(row.call_type, row.request_id, row.spend) for row in rows]}" + ) + search_rows: Final = tuple(row for row in rows if row.call_type == SEARCH_CALL_TYPE) + turn_rows: Final = tuple(row for row in rows if row.call_type != SEARCH_CALL_TYPE) + assert turn_rows, f"session {session_id} carries only search rows: {rows!r}" + assert all(row.session_id == session_id for row in rows), ( + f"session view returned rows outside {session_id}: {[row.session_id for row in rows]}" + ) + assert all((row.spend or 0.0) > 0 for row in search_rows), ( + f"an intercepted search must stay a separately billed row: {[row.spend for row in search_rows]}" + ) + assert {row.request_id for row in search_rows}.isdisjoint({row.request_id for row in turn_rows}), ( + "a search row reused its parent turn's request_id instead of keeping its own: " + f"{[(row.call_type, row.request_id) for row in rows]}" + ) diff --git a/tests/e2e/router/reliability_support.py b/tests/e2e/router/reliability_support.py index 3d5b76f6408..4388e7f11bc 100644 --- a/tests/e2e/router/reliability_support.py +++ b/tests/e2e/router/reliability_support.py @@ -22,6 +22,7 @@ from pydantic import ValidationError from proxy_client import ProxyClient from e2e_config import CHEAP_OPENAI_MODEL, PROXY_BASE_URL, unique_marker from e2e_http import NetworkError, StreamHead, StreamingResponse +from transport import Transport from models import ( CacheControl, ChatMessage, @@ -274,9 +275,36 @@ def chat_turns_override( ) -> StreamingResponse: """POST /chat/completions with an optional per-request router_settings_override, returning the raw outcome so tests read status, body, and reliability headers.""" - return proxy.transport.send( + return chat_turns_override_via( + proxy.transport, key, model, turns, override=override, stream=stream, cache=cache, max_tokens=max_tokens + ) + + +def chat_override_via( + transport: Transport, + key: str, + model: str, + content: str, + override: RouterSettingsOverride | None = None, +) -> StreamingResponse: + """`chat_override` aimed at one replica's transport (from `proxy.replicas`) instead of + the client's default, for cells that must know which gateway took the call.""" + return chat_turns_override_via(transport, key, model, [ChatMessage(role="user", content=content)], override=override) + + +def chat_turns_override_via( + transport: Transport, + key: str, + model: str, + turns: Sequence[ChatMessage], + override: RouterSettingsOverride | None = None, + stream: bool = False, + cache: dict[str, bool] | None = {"no-cache": True}, + max_tokens: int = 512, +) -> StreamingResponse: + return transport.send( "/chat/completions", - headers=proxy.transport.bearer(key), + headers=transport.bearer(key), json=ReliabilityChatBody( model=model, messages=turns, diff --git a/tests/e2e/router/test_reliability_cooldowns_e2e.py b/tests/e2e/router/test_reliability_cooldowns_e2e.py index 769971e1533..ce3bce32880 100644 --- a/tests/e2e/router/test_reliability_cooldowns_e2e.py +++ b/tests/e2e/router/test_reliability_cooldowns_e2e.py @@ -6,21 +6,33 @@ way (a 500, a 429, a 401, or a timeout) holding all of the group's shuffle weigh with an `allowed_fails_policy` of zero for that error class and a short `cooldown_time`, plus a healthy backup at weight 0. The first call, retries off, surfaces the failure to the customer as-is and benches the deployment. The proxy -records the bench off the request path, and a sibling replica only sees it on -its next read of the cooldown keys from Redis, which the cooldown cache does at -most every 1s (DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS). So for +writes the bench to Redis before it answers that failure, and a sibling replica +only sees it on its next read of the cooldown keys from Redis, which the +cooldown cache does at most once per COOLDOWN_REDIS_READ_INTERVAL_SECONDS per +key (DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS, 1s). So for REPLICA_PROPAGATION_SECONDS after the trip, a window kept far wider than that -so this cell asserts the trip and the recovery rather than how fast siblings -catch up, every answer has to be either the deployment's own failure or a 200 -from the backup, which the proxy names in x-litellm-model-id, and at least one -replica has to have served from the backup by then. From then until shortly -before the cooldown can lapse, every call has to land on the backup whichever -replica takes it. Then the test polls until the weighted shuffle opens on the -failing deployment again and the same failure comes back (or, for the 429 pair, -its own 200 once the key's rpm window has reset): that is the recovery, since a -benched deployment is one the router will try again, not one it forgot. Its -deadline counts from the last failure a stale replica caused, because every -failure re-arms the cooldown. +so the trip-then-recover cells assert the trip and the recovery rather than how +fast siblings catch up, every answer has to be either the deployment's own +failure or a 200 from the backup, which the proxy names in x-litellm-model-id, +and at least one replica has to have served from the backup by then. From then +until shortly before the cooldown can lapse, every call has to land on the +backup whichever replica takes it. Then the test polls until the weighted +shuffle opens on the failing deployment again and the same failure comes back +(or, for the 429 pair, its own 200 once the key's rpm window has reset): that +is the recovery, since a benched deployment is one the router will try again, +not one it forgot. Its deadline counts from the last failure a stale replica +caused, because every failure re-arms the cooldown. + +The sibling cell is the one that asserts the speed. It addresses two gateways +from PROXY_REPLICA_URLS by name, warms the second with a healthy call so its +router has already read the failing deployment's cooldown key from Redis and +started the read interval on it, trips the deployment through the first, waits +the interval plus a margin, and then sends the second replica exactly one call, +which has to come back from the backup. One call, because a poll that reached +the failing deployment through the second replica would bench it there too and +hide whether the first replica's bench ever travelled. A stack addressed only +through its load balancer cannot pin which replica takes a call, so the cell is +skipped at collection unless LITELLM_PROXY_REPLICA_URLS names at least two. The failures are the same real ones the retry tests use: a 1ms deadline and a bogus key on the real backend, and this proxy standing in as the upstream for @@ -37,7 +49,7 @@ from dataclasses import dataclass import pytest from complexity_router_client import ComplexityRouterClient -from e2e_config import CHEAP_OPENAI_MODEL, unique_marker +from e2e_config import CHEAP_OPENAI_MODEL, PROXY_REPLICA_URLS, unique_marker from e2e_http import StreamingResponse from lifecycle import ResourceManager from models import KeyGenerateBody, RouterSettingsOverride @@ -45,6 +57,7 @@ from reliability_support import ( COOLDOWN_SECONDS, REPLICA_PROPAGATION_SECONDS, chat_override, + chat_override_via, create_always_5xx_deployment, create_always_rate_limited_deployment, create_always_timing_out_deployment, @@ -54,12 +67,15 @@ from reliability_support import ( model_id_of, spend_only_request_of, ) +from transport import Transport pytestmark = pytest.mark.e2e RECOVERY_GRACE_SECONDS = 10 PROPAGATION_POLL_SECONDS = 0.25 BENCH_MARGIN_SECONDS = 4.0 +COOLDOWN_REDIS_READ_INTERVAL_SECONDS = 1.0 +SIBLING_READ_MARGIN_SECONDS = 1.0 def _call_without_retries(client: ComplexityRouterClient, key: str, group: str) -> StreamingResponse: @@ -68,6 +84,31 @@ def _call_without_retries(client: ComplexityRouterClient, key: str, group: str) ) +def _call_replica_without_retries(transport: Transport, key: str, group: str) -> StreamingResponse: + return chat_override_via( + transport, key, group, f"say hi {unique_marker()}", override=RouterSettingsOverride(num_retries=0) + ) + + +@dataclass(frozen=True, slots=True) +class _Replica: + url: str + transport: Transport + + +def _two_replicas(client: ComplexityRouterClient) -> tuple[_Replica, _Replica]: + first, second, *_ = (_Replica(url, transport) for url, transport in client.proxy.replicas.items()) + return first, second + + +def _warm_cooldown_reads(replica: _Replica, key: str) -> None: + warmed = chat_override_via(replica.transport, key, CHEAP_OPENAI_MODEL, f"say hi {unique_marker()}") + assert warmed.status_code == 200, ( + f"{replica.url} should have answered a healthy {CHEAP_OPENAI_MODEL} call before the trip, got " + f"{warmed.status_code}: {warmed.body[:300]}" + ) + + def _assert_served_by_backup(resp: StreamingResponse, backup: str, when: str) -> None: assert resp.status_code == 200, ( f"{when} the group should have served from the backup, got {resp.status_code}: {resp.body[:300]}" @@ -176,6 +217,47 @@ class TestReliabilityCooldowns: _assert_trips_then_recovers(client, scoped_key, group, failing, backup, failure_status=500) + @pytest.mark.covers("reliability.cooldown.sibling_replica.serves_backup_within_read_interval") + @pytest.mark.skipif( + len(PROXY_REPLICA_URLS) < 2, + reason=( + "this cell trips a deployment through one gateway and reads the bench from another, so " + f"LITELLM_PROXY_REPLICA_URLS has to name at least two distinct gateways, got {PROXY_REPLICA_URLS}" + ), + ) + def test_sibling_replica_serves_backup_within_redis_read_interval( + self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str + ) -> None: + tripping, sibling = _two_replicas(client) + + upstream = f"reliability-cooldown-sibling-upstream-{unique_marker()}" + upstream_id = create_bad_base_deployment(client.proxy, upstream) + resources.defer(lambda: client.proxy.delete_model(upstream_id)) + + group = f"reliability-cooldown-sibling-{unique_marker()}" + failing = create_always_5xx_deployment( + client.proxy, group, upstream, scoped_key, cooldown_time=COOLDOWN_SECONDS + ) + resources.defer(lambda: client.proxy.delete_model(failing)) + backup = create_zero_weight_backup_deployment(client.proxy, group) + resources.defer(lambda: client.proxy.delete_model(backup)) + + _warm_cooldown_reads(sibling, scoped_key) + + tripped = _call_replica_without_retries(tripping.transport, scoped_key, group) + assert tripped.status_code == 500, ( + f"the first call through {tripping.url} should have surfaced the deployment's own 500, got " + f"{tripped.status_code}: {tripped.body[:300]}" + ) + tripped_at = time.monotonic() + + time.sleep(COOLDOWN_REDIS_READ_INTERVAL_SECONDS + SIBLING_READ_MARGIN_SECONDS) + _assert_served_by_backup( + _call_replica_without_retries(sibling.transport, scoped_key, group), + backup, + f"{time.monotonic() - tripped_at:.1f}s after {tripping.url} benched {failing}, on {sibling.url}", + ) + @pytest.mark.covers("reliability.cooldown.429.trips_then_recovers") def test_429_trips_cooldown_then_recovers( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str diff --git a/tests/e2e/router/test_reliability_memory_e2e.py b/tests/e2e/router/test_reliability_memory_e2e.py index 17e3e1a1996..77d3a68cae5 100644 --- a/tests/e2e/router/test_reliability_memory_e2e.py +++ b/tests/e2e/router/test_reliability_memory_e2e.py @@ -38,6 +38,20 @@ size budget: the deterministic catch for a breadcrumb that copies the whole request. It runs before the phases because the leaking writer drops its own rows under the phases' traffic (a queue budget hit, a recursion limit on the nested copies), which would turn the size check into a missing-row check. + +A second, cheaper check holds the idle footprint: every worker's RSS as the harness +read it at collection time, before this pytest process sent any traffic (see +conftest.pytest_collection_finish), must sit under a fixed budget. On the +release gate that is a fresh stack right after its readiness gate, one worker per +gateway replica, so the reading is what a DB-backed boot costs on its own. A +v1.100.x worker with a database idled at 886 MB RSS where v1.101.0rc1 idled at +544 MB on the same database (1.3 GB against 560 MB at the pod level, under a +2 GiB limit): the generated Prisma client at prisma-client-py's default recursive +type depth, 91k TypedDict classes that v1.101.0 cut to 19k with +recursive_type_depth = -1. The budget starts at the rc1 reading plus headroom and +E2E_MEMORY_IDLE_RSS_BUDGET_MB overrides it; a later session on the same stack (the +changed-files workflow's repeat passes, a developer's local loop) measures a proxy +already warmed by traffic, which that headroom also has to cover. """ from __future__ import annotations @@ -55,6 +69,7 @@ import pytest from complexity_router_client import ComplexityRouterClient from e2e_config import ( MEMORY_CONCURRENCY, + MEMORY_IDLE_RSS_BUDGET_MB, MEMORY_REQUESTS_PER_PHASE, MEMORY_RETRIES_PER_REQUEST, MEMORY_RSS_BUDGET_MB, @@ -62,15 +77,16 @@ from e2e_config import ( MEMORY_RSS_SETTLE_SAMPLES, MEMORY_STORED_REQUEST_BUDGET_KB, MEMORY_TRANSCRIPT_TURNS, + PROXY_REPLICA_URLS, unique_marker, ) -from e2e_http import unwrap from lifecycle import ResourceManager +from memory_readings import RssCapture, RssReading, WorkerKey, read_rss_everywhere from models import ChatMessage, RouterSettingsOverride, SpendLogRow from proxy_client import ProxyClient from reliability_support import chat_override, create_never_benched_refusing_deployment -pytestmark = pytest.mark.e2e +pytestmark = [pytest.mark.e2e, pytest.mark.quiet_stack] DEPLOYMENTS_PER_GROUP: Final = 2 RSS_SAMPLE_CAP: Final = 4 * MEMORY_RSS_SETTLE_SAMPLES @@ -84,21 +100,6 @@ class FailedCall: call_id: str | None -WorkerKey = tuple[str, str | None, int] - - -@dataclass(frozen=True, slots=True) -class RssReading: - replica: str - hostname: str | None - worker_pid: int - ram_usage_mb: float - - @property - def worker(self) -> WorkerKey: - return (self.replica, self.hostname, self.worker_pid) - - @dataclass(frozen=True, slots=True) class WorkerGrowth: warm: RssReading @@ -143,12 +144,12 @@ def _fail_many(proxy: ProxyClient, key: str, model: str, override: RouterSetting def _read_rss_everywhere_after_pause(proxy: ProxyClient) -> tuple[RssReading, ...]: time.sleep(MEMORY_RSS_SAMPLE_INTERVAL_SECONDS) - return tuple( - RssReading(replica, body.hostname, body.worker_pid, body.memory.ram_usage_mb) - for replica, result in proxy.memory_summary_everywhere().items() - for body in (unwrap(result),) - if body.memory.ram_usage_mb is not None + capture: Final = read_rss_everywhere(proxy) + assert not capture.failures, ( + f"{len(capture.failures)} replica(s) gave no RSS reading mid-checkpoint, so their workers cannot be " + f"compared with themselves: {'; '.join(capture.failures)}" ) + return capture.readings def _readings_until_no_new_worker( @@ -220,6 +221,26 @@ def _stored_request_kb(proxy: ProxyClient, call: FailedCall) -> float: class TestReliabilityMemory: + @pytest.mark.covers("reliability.perf.idle_memory.under_slo") + def test_workers_idle_under_rss_budget_before_traffic(self, idle_rss: RssCapture) -> None: + assert not idle_rss.failures, ( + f"{len(idle_rss.failures)} replica(s) gave no RSS reading when the session started, so their idle " + f"footprint went unmeasured: {'; '.join(idle_rss.failures)}" + ) + unmeasured: Final = frozenset(PROXY_REPLICA_URLS) - frozenset(reading.replica for reading in idle_rss.readings) + assert not unmeasured, ( + f"{len(unmeasured)} of {len(PROXY_REPLICA_URLS)} replica(s) gave neither an RSS reading nor a failure " + f"reason when the session started, so their idle footprint went unmeasured: {', '.join(sorted(unmeasured))}" + ) + heaviest: Final = idle_rss.heaviest + assert heaviest is not None, "no replica was configured to read, so nothing was measured" + assert heaviest.ram_usage_mb <= MEMORY_IDLE_RSS_BUDGET_MB, ( + f"{heaviest.where} sat at {heaviest.ram_usage_mb:.0f} MB RSS when the session started, before it sent " + f"any traffic, past the {MEMORY_IDLE_RSS_BUDGET_MB:.0f} MB idle budget; a DB-backed v1.100.x worker idled " + f"at 886 MB where v1.101.0rc1 idled at 544 MB, and at that size the release stack's 2 GiB pod limit " + f"leaves the worker little room for traffic" + ) + @pytest.mark.covers("reliability.perf.memory.under_slo") def test_failing_requests_do_not_grow_rss_or_stored_request( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str diff --git a/tests/e2e/stack_lock.py b/tests/e2e/stack_lock.py new file mode 100644 index 00000000000..06df7a20b6a --- /dev/null +++ b/tests/e2e/stack_lock.py @@ -0,0 +1,45 @@ +"""Cross-process reader/writer lock over the proxy stack every xdist worker shares. +Every collected test holds it shared, marker or not, since the Claude Code cells and +other unmarked suites drive the same stack; a `quiet_stack` test holds it exclusive, +and the `gate` file makes a waiting exclusive holder win over readers that arrive +after it.""" + +from __future__ import annotations + +import fcntl +import hashlib +import tempfile +from collections.abc import Generator +from contextlib import ExitStack, contextmanager +from pathlib import Path +from typing import Final + +from e2e_config import PROXY_BASE_URL + +STACK_DIGEST: Final = hashlib.sha256(PROXY_BASE_URL.encode()).hexdigest()[:12] +LOCK_DIR: Final = Path(tempfile.gettempdir()) / f"litellm-e2e-stack-{STACK_DIGEST}" +GATE_FILE: Final = LOCK_DIR / "gate" +STACK_FILE: Final = LOCK_DIR / "stack" + + +@contextmanager +def _flock(path: Path, operation: int) -> Generator[None]: + with path.open("a") as handle: + fcntl.flock(handle, operation) + try: + yield + finally: + fcntl.flock(handle, fcntl.LOCK_UN) + + +@contextmanager +def stack_lock(exclusive: bool) -> Generator[None]: + LOCK_DIR.mkdir(parents=True, exist_ok=True) + if exclusive: + with _flock(GATE_FILE, fcntl.LOCK_EX), _flock(STACK_FILE, fcntl.LOCK_EX): + yield + return + with ExitStack() as held: + with _flock(GATE_FILE, fcntl.LOCK_SH): + held.enter_context(_flock(STACK_FILE, fcntl.LOCK_SH)) + yield diff --git a/tests/e2e/test_proxy_client.py b/tests/e2e/test_proxy_client.py index 0c4aed5bd65..a8f07ed6dd7 100644 --- a/tests/e2e/test_proxy_client.py +++ b/tests/e2e/test_proxy_client.py @@ -338,6 +338,10 @@ class TestParseReplicaUrls: def test_falls_back_to_the_data_plane_address_when_unset(self) -> None: assert parse_replica_urls("", "http://lb") == ("http://lb",) + def test_collapses_repeated_gateway_addresses_to_one_replica(self) -> None: + raw: Final = "http://127.0.0.1:4010,http://127.0.0.1:4010/,http://127.0.0.1:4011,http://127.0.0.1:4010" + assert parse_replica_urls(raw, "http://lb") == ("http://127.0.0.1:4010", "http://127.0.0.1:4011") + def _answers(answers: Iterable[str]) -> ReplicaRead[str]: it: Final = iter(answers) diff --git a/tests/e2e/test_stack_lock.py b/tests/e2e/test_stack_lock.py new file mode 100644 index 00000000000..af071d2cc18 --- /dev/null +++ b/tests/e2e/test_stack_lock.py @@ -0,0 +1,117 @@ +"""Cross-process behavior of the stack lock: readers share it, an exclusive holder waits for +every reader and keeps them out, and a reader arriving behind a waiting exclusive holder +queues behind it instead of starving it.""" + +from __future__ import annotations + +import fcntl +import os +import subprocess +import sys +import time +from contextlib import ExitStack +from pathlib import Path +from typing import Final + +import pytest + +from stack_lock import STACK_DIGEST + +HARNESS_DIR: Final = Path(__file__).resolve().parent +DEADLINE_SECONDS: Final = 30.0 +SETTLE_SECONDS: Final = 0.5 +HOLDER_SCRIPT: Final = """ +import sys, time +from pathlib import Path +from stack_lock import stack_lock +name, mode, release_path, log_path = sys.argv[1:] + + +def record(event): + with Path(log_path).open("a") as log: + log.write(f"{name} {event}\\n") + + +record("waiting") +with stack_lock(exclusive=mode == "exclusive"): + record("enter") + while not Path(release_path).exists(): + time.sleep(0.02) + record("exit") +""" + + +def _events(log_path: Path) -> tuple[str, ...]: + return tuple(log_path.read_text().splitlines()) if log_path.exists() else () + + +def _wait_for_event(log_path: Path, event: str) -> None: + deadline: Final = time.monotonic() + DEADLINE_SECONDS + while event not in _events(log_path): + if time.monotonic() > deadline: + pytest.fail(f"{event!r} never appeared; events so far: {_events(log_path)}") + time.sleep(0.02) + + +def _wait_until_gate_is_held_exclusively(gate_path: Path) -> None: + deadline: Final = time.monotonic() + DEADLINE_SECONDS + with gate_path.open("a") as handle: + while True: + try: + fcntl.flock(handle, fcntl.LOCK_SH | fcntl.LOCK_NB) + except BlockingIOError: + return + fcntl.flock(handle, fcntl.LOCK_UN) + if time.monotonic() > deadline: + pytest.fail("no exclusive holder ever took the gate") + time.sleep(0.02) + + +def _start_holder(held: ExitStack, tmp_path: Path, name: str, mode: str) -> subprocess.Popen[bytes]: + holder: Final = held.enter_context( + subprocess.Popen( + ( + sys.executable, + "-P", + "-c", + HOLDER_SCRIPT, + name, + mode, + str(tmp_path / f"release-{name}"), + str(tmp_path / "events"), + ), + cwd=HARNESS_DIR, + env={**os.environ, "TMPDIR": str(tmp_path), "PYTHONPATH": str(HARNESS_DIR)}, + ) + ) + held.callback(holder.kill) + return holder + + +def test_readers_share_exclusive_waits_and_a_waiting_exclusive_beats_later_readers(tmp_path: Path) -> None: + lock_dir: Final = tmp_path / f"litellm-e2e-stack-{STACK_DIGEST}" + lock_dir.mkdir() + log_path: Final = tmp_path / "events" + with ExitStack() as held: + first_reader: Final = _start_holder(held, tmp_path, "A", "shared") + _wait_for_event(log_path, "A enter") + second_reader: Final = _start_holder(held, tmp_path, "R", "shared") + _wait_for_event(log_path, "R enter") + (tmp_path / "release-R").touch() + _wait_for_event(log_path, "R exit") + writer: Final = _start_holder(held, tmp_path, "W", "exclusive") + _wait_until_gate_is_held_exclusively(lock_dir / "gate") + late_reader: Final = _start_holder(held, tmp_path, "B", "shared") + _wait_for_event(log_path, "B waiting") + time.sleep(SETTLE_SECONDS) + (tmp_path / "release-A").touch() + _wait_for_event(log_path, "W enter") + (tmp_path / "release-W").touch() + _wait_for_event(log_path, "B enter") + (tmp_path / "release-B").touch() + for holder in (first_reader, second_reader, writer, late_reader): + assert holder.wait(timeout=DEADLINE_SECONDS) == 0 + events: Final = _events(log_path) + assert events.index("R enter") < events.index("A exit") + assert events.index("W enter") > events.index("A exit") + assert events.index("B enter") > events.index("W exit") diff --git a/tests/e2e/transport.py b/tests/e2e/transport.py index a3eec815441..87aad0d08de 100644 --- a/tests/e2e/transport.py +++ b/tests/e2e/transport.py @@ -312,7 +312,11 @@ CONTROL_PLANE_PREFIXES: tuple[str, ...] = ( "/global", "/config", "/guardrails", + "/credentials", "/router/settings", + "/audit", + "/public", + "/v2/login", "/openapi.json", ) diff --git a/tests/e2e/ui/helpers/mcp.ts b/tests/e2e/ui/helpers/mcp.ts index 554177e11bc..399f746f3a8 100644 --- a/tests/e2e/ui/helpers/mcp.ts +++ b/tests/e2e/ui/helpers/mcp.ts @@ -1,8 +1,21 @@ import { expect, Page as PwPage } from "@playwright/test"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; import { navigateToPage } from "./navigation"; import { Page } from "../fixtures/pages"; import { masterKey } from "./traffic"; +export async function listUpstreamToolNames(url: string): Promise { + const client = new Client({ name: "litellm-ui-e2e", version: "0.0.0" }); + await client.connect(new StreamableHTTPClientTransport(new URL(url))); + try { + const { tools } = await client.listTools(); + return tools.map((tool) => tool.name); + } finally { + await client.close(); + } +} + /** Creates an MCP server through the UI's discovery to custom-form flow and returns its name. */ export async function createMcpServer(page: PwPage, url: string): Promise { await navigateToPage(page, Page.McpServers); diff --git a/tests/e2e/ui/package-lock.json b/tests/e2e/ui/package-lock.json index b22673a3535..f56e00506e9 100644 --- a/tests/e2e/ui/package-lock.json +++ b/tests/e2e/ui/package-lock.json @@ -8,11 +8,66 @@ "name": "litellm-ui-e2e", "version": "0.0.0", "devDependencies": { + "@modelcontextprotocol/sdk": "1.30.0", "@playwright/test": "1.58.1", "@types/node": "20.19.37", "typescript": "5.9.3" } }, + "node_modules/@hono/node-server": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.1.tgz", + "integrity": "sha512-ELuehkj5VCBdgEw9zs+ivkKwyzzUCSQuE96YmiPvn1ECBoZCczbFXJLeEGMTYjphP6gydh4pHMqEYPVMYUVgQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "hono": "^4" + } + }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.30.0.tgz", + "integrity": "sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@hono/node-server": "^1.19.9 || ^2.0.5", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.2.1", + "express-rate-limit": "^8.2.1", + "hono": "^4.11.4", + "jose": "^6.1.3", + "json-schema-typed": "^8.0.2", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, "node_modules/@playwright/test": { "version": "1.58.1", "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.1.tgz", @@ -39,6 +94,475 @@ "undici-types": "~6.21.0" } }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/body-parser": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "dev": true, + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "dev": true, + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.7.0.tgz", + "integrity": "sha512-hOwV7WOxXfjRpAM1DSJWZDXx3GhplwD8IfwuwvogD8i1Qnkgosw/H45s4ZnFAUHDAhPjlY9hLBvJhKmGMyY26g==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", + "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -54,6 +578,396 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hono": { + "version": "4.13.8", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.8.tgz", + "integrity": "sha512-/Gng7NfoykZl2pjukW5Z6+8Yxm3BPRf86GTbQnt0SbySkvax4fyL4H3HhY1cCpBGmiW9XDRFzRV+CXK2W8QudQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.9.0" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/ip-address": { + "version": "10.7.2", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.2.tgz", + "integrity": "sha512-7H/2gFSIitxc0hG3nOI1glS8QLo/EHBFFLk8vEUjXY/xu0AdL8jZ9U1IzO2PUm0d2D/ofQcAifb0g6OBkt8U7w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.2.12", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz", + "integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-typed": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", + "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz", + "integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "content-type": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/negotiator/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "dev": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/playwright": { "version": "1.58.1", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.1.tgz", @@ -86,6 +1000,311 @@ "node": ">=18" } }, + "node_modules/proxy-addr": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", + "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/qs": { + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", + "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "dev": true, + "license": "ISC" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "dev": true, + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -106,6 +1325,69 @@ "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", "dev": true, "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/zod": { + "version": "4.6.5", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", + "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "dev": true, + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } } } } diff --git a/tests/e2e/ui/package.json b/tests/e2e/ui/package.json index ede759d97cb..78130412be9 100644 --- a/tests/e2e/ui/package.json +++ b/tests/e2e/ui/package.json @@ -9,6 +9,7 @@ "e2e:migration:root": "playwright test --config migration.serverRootPath.config.ts" }, "devDependencies": { + "@modelcontextprotocol/sdk": "1.30.0", "@playwright/test": "1.58.1", "@types/node": "20.19.37", "typescript": "5.9.3" diff --git a/tests/e2e/ui/tests/mcp/mcpTools.spec.ts b/tests/e2e/ui/tests/mcp/mcpTools.spec.ts index 225ca8b9449..2390a78755e 100644 --- a/tests/e2e/ui/tests/mcp/mcpTools.spec.ts +++ b/tests/e2e/ui/tests/mcp/mcpTools.spec.ts @@ -1,17 +1,20 @@ import { test, expect, Locator } from "@playwright/test"; import { ADMIN_STORAGE_PATH } from "../../constants"; -import { createMcpServer, deleteMcpServerByName, openMcpToolsTab } from "../../helpers/mcp"; +import { createMcpServer, deleteMcpServerByName, listUpstreamToolNames, openMcpToolsTab } from "../../helpers/mcp"; // Listing and calling MCP tools, which needs a server that really answers; the create-only spec // points at an unreachable URL on purpose. // -// This spec makes a read-only network call to DeepWiki's public MCP server, from the proxy rather -// than the browser. It needs no credentials, so there is no secret to leak from a public repo. +// This spec makes read-only network calls to DeepWiki's public MCP server: from the proxy, and from +// the test runner to learn which tools the upstream advertises today, so the tool list is never +// pinned here. It needs no credentials, so there is no secret to leak from a public repo. // // A DeepWiki outage turns this red for something that is not a litellm regression. That is left // visible rather than auto-skipped: skipping on connection trouble also skips when the proxy's own // MCP client breaks, which is the regression this exists to catch. E2E_SKIP_EXTERNAL_MCP=1 opts out. const MCP_SERVER_URL = "https://mcp.deepwiki.com/mcp"; +// Read from DeepWiki's tools/list on 2026-09-22. One name has to be pinned so the call-tool test can +// fill a known input (repoName); the listing test checks it is still advertised before the UI checks. const TOOL_NAME = "read_wiki_structure"; const TOOL_ARG_REPO = "BerriAI/litellm"; @@ -36,14 +39,17 @@ test.describe("MCP Tools", () => { }); test("MCP Tools tab lists the tools the upstream server advertises", async ({ page }) => { + const upstreamTools = await listUpstreamToolNames(MCP_SERVER_URL); + expect(upstreamTools).toContain(TOOL_NAME); + // Fetched through the proxy on mount, so allow for a cold upstream connection. const toolList = page.locator(".mcp-tools-scrollable"); await expect(toolList).toBeVisible({ timeout: 30_000 }); - // Non-empty would still pass if the proxy returned some other server's tools. - await expect(toolCard(toolList, TOOL_NAME)).toBeVisible(); - await expect(toolCard(toolList, "ask_question")).toBeVisible(); - await expect(toolCard(toolList, "read_wiki_contents")).toBeVisible(); + for (const name of upstreamTools) { + await expect(toolCard(toolList, name)).toBeVisible(); + } + await expect(toolList.locator("h4.font-mono")).toHaveCount(upstreamTools.length); // No other tool's name or description contains this string, so exactly one card survives. await page.getByPlaceholder("Search tools...").fill(TOOL_NAME); diff --git a/tests/integration/_support/wire.py b/tests/integration/_support/wire.py index acc51dd4497..0c6acfde96c 100644 --- a/tests/integration/_support/wire.py +++ b/tests/integration/_support/wire.py @@ -1,7 +1,8 @@ from __future__ import annotations +import ssl import threading -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Generator, Mapping from contextlib import contextmanager from dataclasses import dataclass from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -38,7 +39,7 @@ class Wire: @contextmanager -def wire_server(respond: Callable[[Request], Reply]) -> Iterator[Wire]: +def wire_server(respond: Callable[[Request], Reply], tls: ssl.SSLContext | None = None) -> Generator[Wire, None, None]: """Owned TCP peer; requests traverse the real HTTP client and serialization.""" received: Final[SimpleQueue[Request]] = SimpleQueue() errors: Final[SimpleQueue[Exception]] = SimpleQueue() @@ -50,7 +51,8 @@ def wire_server(respond: Callable[[Request], Reply]) -> Iterator[Wire]: def respond(self) -> None: request: Final = Request( - self.command, self.path, + self.command, + self.path, {name.lower(): value for name, value in self.headers.items()}, self.rfile.read(int(self.headers.get("content-length", "0"))), ) @@ -99,11 +101,20 @@ def wire_server(respond: Callable[[Request], Reply]) -> Iterator[Wire]: class OwnedHTTPServer(ThreadingHTTPServer): daemon_threads = False + def server_bind(self) -> None: + super().server_bind() + if tls is not None: + self.socket = tls.wrap_socket(self.socket, server_side=True) + with OwnedHTTPServer(("127.0.0.1", 0), Handler) as server: thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) thread.start() try: - yield Wire(f"http://127.0.0.1:{server.server_port}", received, disconnected) + yield Wire( + f"{'https' if tls is not None else 'http'}://127.0.0.1:{server.server_port}", + received, + disconnected, + ) finally: server.shutdown() thread.join(timeout=6) diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 6a085fb7f3d..a1c018a8864 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -172,18 +172,72 @@ "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_video_failed_result_reports_failed_status_and_fal_error": [ "other.provider_wire.fal_ai.video_failed_result_surfaces_fal_error" ], + "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_auto_duration_omits_duration_and_queues": [ + "other.provider_wire.fal_ai.h3_auto_duration_omits_duration_and_queues" + ], + "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_oversized_size_uses_top_resolution_tier_and_queues": [ + "other.provider_wire.fal_ai.h3_oversized_size_uses_top_resolution_tier" + ], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_generation_sends_quality_and_size_and_charges_keyed_row": [ "other.provider_wire.fal_ai.gpt_image_generation_quality_size_wire_and_keyed_pricing" ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_generation_prices_non_canonical_size_from_nearest_row": [ + "other.provider_wire.fal_ai.gpt_image_generation_noncanonical_size_uses_nearest_keyed_row" + ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_sdk_response_honors_dump_options": [ + "other.provider_wire.fal_ai.sdk_image_response_dump_options" + ], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_dev_generation_targets_dev_endpoint_and_charges_per_image": [ "other.provider_wire.fal_ai.flux_dev_endpoint_and_per_image_pricing" ], + "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_charges_and_polls_pass_through_free": [ + "other.provider_wire.fal_ai.passthrough_queue_submit_charges_and_polls_do_not" + ], + "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_prices_string_resolution_like_the_integer": [ + "other.provider_wire.fal_ai.passthrough_queue_submit_prices_string_resolution_like_integer" + ], + "tests/integration/providers/test_fal_ai_passthrough_wire.py::test_fal_queue_submit_to_catalog_key_the_pricer_cannot_price_is_rejected_not_forwarded": [ + "other.provider_wire.fal_ai.passthrough_queue_submit_rejects_unpriceable_catalog_key" + ], "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_gpt_image_25_edit_inlines_upload_as_data_url_and_charges_keyed_row": [ "other.provider_wire.fal_ai.image_edit_json_data_urls_and_keyed_pricing" ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_lora_depth_edit_sends_single_image_url_and_charges_flat_row": [ + "other.provider_wire.fal_ai.flux_lora_depth_edit_single_image_url_and_flat_pricing" + ], + "tests/integration/providers/test_fal_ai_chat_wire.py::test_fal_moondream3_chat_sends_prompt_image_and_reasoning": [ + "other.provider_wire.fal_ai.moondream3_chat_query_wire_and_token_pricing" + ], + "tests/integration/providers/test_fal_ai_chat_wire.py::test_fal_moondream3_chat_rejects_non_string_reasoning_effort_before_the_wire": [ + "other.provider_wire.fal_ai.chat_non_string_reasoning_effort_rejected_before_wire" + ], + "tests/integration/providers/test_fal_ai_image_wire.py::test_fal_flux_dev_generation_without_deployment_api_base_uses_global_api_base": [ + "other.provider_wire.fal_ai.global_api_base_routes_image_generation" + ], + "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price[mimo-v2.6-pro]": [ + "other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing" + ], + "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price[mimo-v2.6-flash]": [ + "other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing" + ], + "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_stream_delivers_reasoning_then_answer_deltas": [ + "other.provider_wire.xiaomi_mimo.reasoning_and_answer_stream_as_deltas" + ], + "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_tool_call_is_forwarded_and_returned": [ + "other.provider_wire.xiaomi_mimo.tool_call_survives_translation" + ], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_video_create_uses_canonical_body_and_status_path": [ "other.provider_wire.fal_ai.video_queue_create_status_and_content_download" ], + "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_result_probe_carries_the_deployment_extra_headers": [ + "other.provider_wire.fal_ai.video_result_probe_forwards_extra_headers" + ], + "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_result_probe_reuses_the_ssl_verify_false_client": [ + "other.provider_wire.fal_ai.video_result_probe_honors_ssl_verify" + ], + "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_provider_hanging_up_on_the_result_probe_keeps_the_completed_status": [ + "other.provider_wire.fal_ai.video_result_probe_hangup_stays_completed" + ], "tests/integration/mcp/test_mcp_lifecycle.py::test_saved_headers_reach_real_mcp_tool_and_survive_unrelated_edit": [ "mcp.call_tool.saved_headers.reach_actual_transport" ], @@ -197,6 +251,9 @@ "other.observability.callbacks.credentials_stay_out_of_event_bodies", "other.observability.callbacks.concurrent_results_join_complete_events_and_rows" ], + "tests/integration/observability/test_otel_text_completion_choices.py::test_otel_weave_output_keeps_text_completion_provider_fields_beside_the_synthesized_message": [ + "other.observability.otel.text_completion_choices_keep_provider_fields" + ], "tests/integration/observability/test_guardrail_effects.py::test_guardrail_rewrites_system_and_user_in_actual_anthropic_request": [ "other.observability.guardrails.rewrite_reaches_correct_anthropic_positions" ], diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index ea8bf230d05..ac3fcd33d2e 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from collections.abc import Mapping from pathlib import Path from types import MappingProxyType @@ -8,6 +9,7 @@ from typing import Annotated, Final, Literal, TypeAlias from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator CASES_PATH: Final = Path(__file__).resolve().parent / "cost_tracking_cases.json" +PRIOR_RESPONSE_ID_MARKER: Final = "$PRIOR_RESPONSE_ID" class SearchContextCostPerQuery(BaseModel): @@ -312,6 +314,21 @@ class CostTrackingTestCase(BaseModel): usage: Final = self.response.body.get("usage") return isinstance(usage, dict) and isinstance(usage.get("cost"), (int, float)) + @property + def chains_prior_response(self) -> bool: + return self.request.get("previous_response_id") == PRIOR_RESPONSE_ID_MARKER + + @property + def can_chain_prior_response(self) -> bool: + return ( + self.chains_prior_response + and self.endpoint == "/v1/responses" + and isinstance(self.response, JsonResponse) + and isinstance(self.response.body.get("id"), str) + and not isinstance(self.expected, FailureExpected) + and not (isinstance(self.expected, ExactExpected) and self.expected.rollups) + ) + class BatchOutputLine(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") @@ -681,6 +698,11 @@ def data_errors() -> tuple[str, ...]: for marker in ('"id": "call_$REQUEST_ID"', '"id": "toolu_$REQUEST_ID"') ) ) + invalid_prior_response_chains: Final = sorted( + case.name + for case in CASES + if PRIOR_RESPONSE_ID_MARKER in json.dumps(case.request) and not case.can_chain_prior_response + ) return tuple( message for message in ( @@ -700,6 +722,10 @@ def data_errors() -> tuple[str, ...]: f"pinned tool IDs contain $REQUEST_ID: {invalid_pinned_tool_ids}" if invalid_pinned_tool_ids else None, + f"{PRIOR_RESPONSE_ID_MARKER} needs a non-rollup, non-failure /v1/responses JSON response with a string id" + f" as previous_response_id: {invalid_prior_response_chains}" + if invalid_prior_response_chains + else None, ) if message is not None ) diff --git a/tests/integration/cost_calculation/cost_tracking_cases.json b/tests/integration/cost_calculation/cost_tracking_cases.json index 17ebc793fae..09dfa66012f 100644 --- a/tests/integration/cost_calculation/cost_tracking_cases.json +++ b/tests/integration/cost_calculation/cost_tracking_cases.json @@ -27733,7 +27733,7 @@ "request": { "model": "$MODEL", "input": "continue this text", - "previous_response_id": "resp_scripted_prior" + "previous_response_id": "$PRIOR_RESPONSE_ID" }, "response": { "content_type": "application/json", diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py index efab17acba4..82878634677 100644 --- a/tests/integration/cost_calculation/test_cost_tracking.py +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -9,13 +9,14 @@ import time import uuid import wave import zlib +from collections.abc import Mapping from hashlib import sha256 from itertools import islice from typing import Final, cast import httpx import pytest -from integration._support.client import JSON_OBJECT, Gateway +from integration._support.client import JSON_OBJECT, Gateway, string_value from integration._support.upstream import delete_scenario, register_scenario from integration.cost_calculation.assertions import assert_exact, assert_recount from integration.cost_calculation.conftest import ( @@ -112,6 +113,19 @@ def _replace_model(value: JsonValue, model_name: str) -> JsonValue: return value +def _prime_prior_response( + gateway: Gateway, request_path: str, request_values: Mapping[str, JsonValue], key: str +) -> str: + primed: Final = gateway.request( + "POST", + request_path, + {field: value for field, value in request_values.items() if field != "previous_response_id"}, + key=key, + ) + assert primed.is_success, f"priming response failed: {primed.status_code}: {primed.text[:400]}" + return string_value(JSON_OBJECT.validate_json(primed.content)["id"]) + + @pytest.mark.parametrize("case", _CASES) def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) -> None: marker: Final = sha256(case.name.encode()).hexdigest()[:12] @@ -175,21 +189,6 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) if isinstance(expected, ExactExpected) and expected.rollups else None ) - request_body: Final = JSON_OBJECT.validate_python( - { - **base_request_values, - **( - {"model": fallback_deployment.model_name, "fallbacks": [model_name]} - if fallback_deployment is not None - else {} - ), - **( - {"user": end_user_id, "cache": {"no-cache": True}} - if end_user_id is not None - else {} - ), - } - ) request_headers: Final = ( { "x-pass-x-scripted-scenario": scenario_id, @@ -207,6 +206,27 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) if passthrough_provider is not None else case.endpoint ) + prior_response_id: Final = ( + _prime_prior_response(gateway, request_path, base_request_values, key) + if case.chains_prior_response + else None + ) + request_body: Final = JSON_OBJECT.validate_python( + { + **base_request_values, + **( + {"model": fallback_deployment.model_name, "fallbacks": [model_name]} + if fallback_deployment is not None + else {} + ), + **( + {"user": end_user_id, "cache": {"no-cache": True}} + if end_user_id is not None + else {} + ), + **({"previous_response_id": prior_response_id} if prior_response_id is not None else {}), + } + ) if case.disconnect_after_frames is not None: with gateway.client.stream( "POST", @@ -250,7 +270,7 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) assert response.is_success, f"{case.name}: proxy returned {response.status_code}: {response.text[:400]}" if case.response.content_type == "text/event-stream": _assert_stream_has_no_error(response.text) - rows: Final = poll_rows(key, len(responses)) + rows: Final = poll_rows(key, len(responses) + (prior_response_id is not None)) if isinstance(expected, RecountExpected): row: Final = rows[0] assert_recount(case.name, expected, row) diff --git a/tests/integration/observability/test_otel_text_completion_choices.py b/tests/integration/observability/test_otel_text_completion_choices.py new file mode 100644 index 00000000000..70c104a61aa --- /dev/null +++ b/tests/integration/observability/test_otel_text_completion_choices.py @@ -0,0 +1,110 @@ +import json +import uuid +from pathlib import Path +from typing import Final + +import pytest +import yaml + +from integration._support.client import Gateway, eventually +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + + +def _span_attributes(body: bytes) -> tuple[dict[str, object], ...]: + return tuple( + {attribute["key"]: attribute["value"] for attribute in span.get("attributes", ())} + for resource in json.loads(body)["resourceSpans"] + for scope in resource["scopeSpans"] + for span in scope["spans"] + ) + + +@pytest.mark.covers("other.observability.otel.text_completion_choices_keep_provider_fields") +def test_otel_weave_output_keeps_text_completion_provider_fields_beside_the_synthesized_message( + gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = "otel-text-" + uuid.uuid4().hex + logprobs: Final = { + "tokens": ["Hello", " there"], + "token_logprobs": [-0.1, -0.2], + "top_logprobs": None, + "text_offset": [0, 5], + } + content_filter: Final = {"hate": {"filtered": False, "severity": "safe"}} + + def upstream(request: Request) -> Reply: + assert request.target.endswith("/completions"), request.target + return Reply( + body=json.dumps( + { + "id": marker, + "object": "text_completion", + "created": 1, + "model": "gpt-3.5-turbo-instruct", + "choices": [ + { + "index": 0, + "text": "Hello there", + "finish_reason": "stop", + "logprobs": logprobs, + "content_filter_results": content_filter, + } + ], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, + } + ).encode() + ) + + def sink(_request: Request) -> Reply: + return Reply() + + with wire_server(upstream) as provider, wire_server(sink) as collector: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["litellm_settings"].update({"callbacks": ["otel"]}) + config["callback_settings"] = { + "otel": { + "exporter": "http/json", + "endpoint": collector.url, + "mapper_names": ["genai", "openinference", "weave"], + "capture_message_content": "span_only", + "use_simple_processor": True, + } + } + path: Final = tmp_path / "otel.yaml" + path.write_text(yaml.safe_dump(config)) + with ( + owned_proxy(gateway, tmp_path, {"LITELLM_OTEL_V2": "1"}, config=path) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model(model="openai/gpt-3.5-turbo-instruct", api_base=provider.url + "/v1") + response: Final = candidate.request( + "POST", "/v1/completions", {"model": model, "prompt": marker, "cache": {"no-cache": True}} + ) + assert response.status_code == 200, response.text + assert response.json()["choices"][0]["text"] == "Hello there" + batches = [] + + def outputs() -> tuple[list[dict[str, object]], ...]: + batches.extend(collector.drain()) + return tuple( + json.loads(attributes["weave.output"]["stringValue"]) + for batch in batches + for attributes in _span_attributes(batch.body) + if "weave.output" in attributes + and attributes.get("gen_ai.response.id", {}).get("stringValue") == marker + ) + + choices: Final = eventually(outputs, lambda values: len(values) == 1, seconds=20)[0] + assert len(choices) == 1, choices + choice: Final = choices[0] + assert choice["message"]["content"] == "Hello there", choice + assert "text" not in choice, choice + assert { + key: choice.get(key) for key in ("index", "finish_reason", "logprobs", "content_filter_results") + } == { + "index": 0, + "finish_reason": "stop", + "logprobs": logprobs, + "content_filter_results": content_filter, + }, choice diff --git a/tests/integration/providers/test_fal_ai_chat_wire.py b/tests/integration/providers/test_fal_ai_chat_wire.py new file mode 100644 index 00000000000..8cfa51f4389 --- /dev/null +++ b/tests/integration/providers/test_fal_ai_chat_wire.py @@ -0,0 +1,126 @@ +import json +from pathlib import Path +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_MODEL: Final = "fal-ai/moondream3-preview/query" +_PROMPT: Final = "what is in this image?" +_COST_MAP_PATH: Final = Path(__file__).resolve().parents[3] / "model_prices_and_context_window.json" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_COST_MAP: Final = TypeAdapter(dict[str, dict[str, object]]) + + +def _catalog_cost(key: str, field: str) -> float: + cost_map: Final = _COST_MAP.validate_json(_COST_MAP_PATH.read_bytes()) + cost_value: Final = cost_map[key][field] + assert isinstance(cost_value, (int, float)) + return float(cost_value) + + +def _approx(value: float) -> object: + return pytest.approx(value, rel=1e-6) # pyright: ignore[reportUnknownMemberType] # pytest lacks typed approx stubs + + +@pytest.mark.covers("other.provider_wire.fal_ai.moondream3_chat_query_wire_and_token_pricing") +def test_fal_moondream3_chat_sends_prompt_image_and_reasoning(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.headers["authorization"] == "Key synthetic-fal-key" + assert request.target == f"/{_MODEL}" + assert request.headers["content-type"] == "application/json" + assert _JSON_OBJECT.validate_json(request.body) == { + "prompt": _PROMPT, + "image_url": "https://example.com/pic.png", + "reasoning": False, + "temperature": 0.2, + } + return Reply( + body=json.dumps( + { + "output": "a red circle on a blue background", + "reasoning": "inspected the shapes", + "finish_reason": "stop", + "usage_info": { + "input_tokens": 11, + "output_tokens": 7, + "prefill_time_ms": 1.0, + "decode_time_ms": 2.0, + "ttft_ms": 1.5, + }, + } + ).encode() + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"fal_ai/{_MODEL}", api_base=wire.url, api_key="synthetic-fal-key") + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": _PROMPT}, + {"type": "image_url", "image_url": {"url": "https://example.com/pic.png"}}, + ], + } + ], + "reasoning_effort": "none", + "temperature": 0.2, + }, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["choices"] == [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "role": "assistant", + "content": "a red circle on a blue background", + "reasoning_content": "inspected the shapes", + }, + } + ] + assert payload["usage"] == {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18} + cost: Final = float(response.headers["x-litellm-response-cost"]) + assert cost == _approx( + 11 * _catalog_cost(f"fal_ai/{_MODEL}", "input_cost_per_token") + + 7 * _catalog_cost(f"fal_ai/{_MODEL}", "output_cost_per_token") + ) + assert [(request.method, request.target) for request in wire.drain()] == [("POST", f"/{_MODEL}")] + + +@pytest.mark.covers("other.provider_wire.fal_ai.chat_non_string_reasoning_effort_rejected_before_wire") +def test_fal_moondream3_chat_rejects_non_string_reasoning_effort_before_the_wire(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + raise AssertionError(f"provider must not be reached: {request.method} {request.target}") + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"fal_ai/{_MODEL}", api_base=wire.url, api_key="synthetic-fal-key") + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": _PROMPT}, + {"type": "image_url", "image_url": {"url": "https://example.com/pic.png"}}, + ], + } + ], + "reasoning_effort": {"level": "low"}, + }, + ) + assert response.status_code == 400, response.text + assert "reasoning_effort" in response.text + assert wire.drain() == () diff --git a/tests/integration/providers/test_fal_ai_image_wire.py b/tests/integration/providers/test_fal_ai_image_wire.py index f9ceac0b037..c85bcdae8b8 100644 --- a/tests/integration/providers/test_fal_ai_image_wire.py +++ b/tests/integration/providers/test_fal_ai_image_wire.py @@ -5,10 +5,14 @@ from typing import Final import httpx import pytest -from integration._support.client import Gateway +import yaml +from integration._support.client import Gateway, object_value +from integration._support.process import owned_proxy from integration._support.wire import Reply, Request, wire_server from pydantic import JsonValue, TypeAdapter +import litellm + _GPT_IMAGE_MODEL: Final = "openai/gpt-image-2.5/flare/text-to-image" _FLUX_MODEL: Final = "fal-ai/flux/dev" _EDIT_MODEL: Final = "openai/gpt-image-2.5/flare/edit" @@ -120,6 +124,63 @@ def test_fal_gpt_image_25_generation_sends_quality_and_size_and_charges_keyed_ro ] +@pytest.mark.covers("other.provider_wire.fal_ai.gpt_image_generation_noncanonical_size_uses_nearest_keyed_row") +def test_fal_gpt_image_25_generation_prices_non_canonical_size_from_nearest_row(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.headers["authorization"] == "Key synthetic-fal-key" + assert request.target == "/openai/gpt-image-2.5/flare/text-to-image" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body == {"prompt": _PROMPT, "quality": "low", "image_size": {"width": 1536, "height": 1024}} + return Reply(body=_image_response(((f"{wire_url}/files/noncanonical.png", 1536, 1024),), _PROMPT)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + wire_url: Final = wire.url + model: Final = scenario.model( + model=f"fal_ai/{_GPT_IMAGE_MODEL}", api_base=wire.url, api_key="synthetic-fal-key" + ) + response: Final = gateway.request( + "POST", + "/v1/images/generations", + {"model": model, "prompt": _PROMPT, "quality": "low", "size": "1536x1024"}, + ) + assert response.status_code == 200, response.text + cost: Final = _response_cost(response) + assert cost == _approx(_catalog_cost("fal_ai/low/1024-x-1536/openai/gpt-image-2.5/flare/text-to-image")) + assert [(request.method, request.target) for request in wire.drain()] == [ + ("POST", "/openai/gpt-image-2.5/flare/text-to-image") + ] + + +@pytest.mark.covers("other.provider_wire.fal_ai.sdk_image_response_dump_options") +def test_fal_gpt_image_sdk_response_honors_dump_options() -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.headers["authorization"] == "Key synthetic-fal-key" + assert request.target == "/openai/gpt-image-2.5/flare/text-to-image" + assert _JSON_OBJECT.validate_json(request.body) == {"prompt": _PROMPT, "quality": "low"} + return Reply(body=_image_response((("https://example.com/fal.png", 1024, 1536),), _PROMPT)) + + with wire_server(respond) as wire: + response: Final = litellm.image_generation( + model=_GPT_IMAGE_MODEL, + prompt=_PROMPT, + quality="low", + api_base=wire.url, + api_key="synthetic-fal-key", + custom_llm_provider="fal_ai", + ) + assert response.model_dump(exclude_none=True)["data"] == [ + { + "url": "https://example.com/fal.png", + "provider_specific_fields": {"width": 1024, "height": 1536, "content_type": "image/png"}, + } + ] + assert [(request.method, request.target) for request in wire.drain()] == [ + ("POST", "/openai/gpt-image-2.5/flare/text-to-image") + ] + + @pytest.mark.covers("other.provider_wire.fal_ai.flux_dev_endpoint_and_per_image_pricing") def test_fal_flux_dev_generation_targets_dev_endpoint_and_charges_per_image(gateway: Gateway) -> None: def respond(request: Request) -> Reply: @@ -205,3 +266,82 @@ def test_fal_gpt_image_25_edit_inlines_upload_as_data_url_and_charges_keyed_row( assert [(request.method, request.target) for request in wire.drain()] == [ ("POST", "/openai/gpt-image-2.5/flare/edit") ] + + +@pytest.mark.covers("other.provider_wire.fal_ai.flux_lora_depth_edit_single_image_url_and_flat_pricing") +def test_fal_flux_lora_depth_edit_sends_single_image_url_and_charges_flat_row(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.headers["authorization"] == "Key synthetic-fal-key" + assert request.target == "/fal-ai/flux-lora-depth" + assert request.headers["content-type"] == "application/json" + assert _JSON_OBJECT.validate_json(request.body) == { + "prompt": _PROMPT, + "image_url": "data:image/png;base64," + base64.b64encode(_PNG_BYTES).decode(), + } + return Reply(body=_image_response(((f"{wire_url}/files/depth.png", 1024, 1024),), _PROMPT)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + wire_url: Final = wire.url + model: Final = scenario.model( + model="fal_ai/fal-ai/flux-lora-depth", api_base=wire.url, api_key="synthetic-fal-key" + ) + response: Final = gateway.client.post( + "/v1/images/edits", + data={"model": model, "prompt": _PROMPT}, + files={"image": ("red_circle.png", _PNG_BYTES, "image/png")}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["data"] == [ + { + "url": f"{wire.url}/files/depth.png", + "b64_json": None, + "revised_prompt": None, + "provider_specific_fields": {"width": 1024, "height": 1024, "content_type": "image/png"}, + } + ] + cost: Final = _response_cost(response) + assert cost == _approx(_catalog_cost("fal_ai/fal-ai/flux-lora-depth")) + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/fal-ai/flux-lora-depth")] + + +@pytest.mark.covers("other.provider_wire.fal_ai.global_api_base_routes_image_generation") +def test_fal_flux_dev_generation_without_deployment_api_base_uses_global_api_base( + gateway: Gateway, tmp_path: Path +) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.headers["authorization"] == "Key synthetic-fal-key" + assert request.target == "/fal-ai/flux/dev" + assert _JSON_OBJECT.validate_json(request.body) == {"prompt": _PROMPT, "num_images": 1} + return Reply(body=_image_response(((f"{wire_url}/files/global.png", 1024, 1024),), _PROMPT)) + + with wire_server(respond) as wire: + wire_url: Final = wire.url + configuration: Final = _JSON_OBJECT.validate_python( + yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + ) + configuration["litellm_settings"] = { + **object_value(configuration["litellm_settings"]), + "api_base": wire.url, + } + path: Final = tmp_path / "global-api-base.yaml" + path.write_text(yaml.safe_dump(configuration)) + with owned_proxy(gateway, tmp_path, {}, config=path) as candidate, candidate.scenario() as scenario: + model: Final = scenario.model(model=f"fal_ai/{_FLUX_MODEL}", api_key="synthetic-fal-key", api_base=None) + response: Final = candidate.request( + "POST", "/v1/images/generations", {"model": model, "prompt": _PROMPT, "n": 1} + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["data"] == [ + { + "url": f"{wire.url}/files/global.png", + "b64_json": None, + "revised_prompt": None, + "provider_specific_fields": {"width": 1024, "height": 1024, "content_type": "image/png"}, + } + ] + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/fal-ai/flux/dev")] diff --git a/tests/integration/providers/test_fal_ai_passthrough_wire.py b/tests/integration/providers/test_fal_ai_passthrough_wire.py new file mode 100644 index 00000000000..b80cc601b3b --- /dev/null +++ b/tests/integration/providers/test_fal_ai_passthrough_wire.py @@ -0,0 +1,191 @@ +import json +from pathlib import Path +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + +_MODEL: Final = "fal-ai/trellis-2" +_REQUEST_BODY: Final = {"image_url": "https://example.com/in.png", "resolution": 1536} +_UPSTREAM_BODY: Final = { + "model_glb": { + "url": "https://fal.media/model.glb", + "content_type": "model/gltf-binary", + "file_name": "model.glb", + "file_size": 123, + } +} +_EXPECTED_SPEND: Final = 0.35 + + +@pytest.mark.covers("other.provider_wire.fal_ai.passthrough_queue_submit_charges_and_polls_do_not") +def test_fal_queue_submit_charges_and_polls_pass_through_free(gateway: Gateway, tmp_path) -> None: + def respond(request: Request) -> Reply: + assert request.headers["authorization"] == "Key synthetic-fal-key" + if request.method == "POST": + assert request.target == f"/{_MODEL}" + assert json.loads(request.body) == _REQUEST_BODY + return Reply(body=json.dumps({"request_id": "req-1", "status": "IN_QUEUE"}).encode()) + if request.target == f"/{_MODEL}/requests/req-1/status": + return Reply(body=json.dumps({"status": "COMPLETED"}).encode()) + assert request.target == f"/{_MODEL}/requests/req-1" + return Reply(body=json.dumps(_UPSTREAM_BODY).encode()) + + config: Final = tmp_path / "proxy_config.yaml" + config.write_text( + "model_list: []\n" + "general_settings:\n" + " master_key: os.environ/LITELLM_MASTER_KEY\n" + " database_url: os.environ/DATABASE_URL\n" + " store_model_in_db: true\n" + " disable_spend_logs: false\n" + " proxy_batch_write_at: 1\n" + "router_settings:\n" + " disable_cooldowns: true\n" + ) + with wire_server(respond) as wire: + with owned_proxy( + gateway, + tmp_path, + {"FAL_AI_QUEUE_API_BASE": wire.url, "FAL_AI_API_KEY": "synthetic-fal-key"}, + config=config, + ) as candidate: + submit: Final = candidate.request("POST", f"/fal_ai/{_MODEL}", _REQUEST_BODY) + assert submit.status_code == 200, submit.text + assert json.loads(submit.content) == {"request_id": "req-1", "status": "IN_QUEUE"} + status_response: Final = candidate.request("GET", f"/fal_ai/{_MODEL}/requests/req-1/status") + assert status_response.status_code == 200, status_response.text + assert json.loads(status_response.content) == {"status": "COMPLETED"} + result_response: Final = candidate.request("GET", f"/fal_ai/{_MODEL}/requests/req-1") + assert result_response.status_code == 200, result_response.text + assert json.loads(result_response.content) == _UPSTREAM_BODY + submit_spend: Final = eventually( + lambda: read_rows( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (submit.headers["x-litellm-call-id"],), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert float(submit_spend[0]["spend"]) == pytest.approx(_EXPECTED_SPEND) + poll_rows: Final = eventually( + lambda: read_rows( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=ANY(%s)', + ([status_response.headers["x-litellm-call-id"], result_response.headers["x-litellm-call-id"]],), + ), + lambda values: len(values) == 2, + seconds=70, + ) + assert sorted(float(row["spend"]) for row in poll_rows) == [0.0, 0.0] + assert [(request.method, request.target) for request in wire.drain()] == [ + ("POST", f"/{_MODEL}"), + ("GET", f"/{_MODEL}/requests/req-1/status"), + ("GET", f"/{_MODEL}/requests/req-1"), + ] + + +@pytest.mark.covers("other.provider_wire.fal_ai.passthrough_queue_submit_rejects_unpriceable_catalog_key") +def test_fal_queue_submit_to_catalog_key_the_pricer_cannot_price_is_rejected_not_forwarded( + gateway: Gateway, tmp_path: Path +) -> None: + def respond(request: Request) -> Reply: + return Reply(body=json.dumps({"request_id": "req-1", "status": "IN_QUEUE"}).encode()) + + config: Final = tmp_path / "proxy_config.yaml" + config.write_text( + "model_list: []\n" + "general_settings:\n" + " master_key: os.environ/LITELLM_MASTER_KEY\n" + " database_url: os.environ/DATABASE_URL\n" + " store_model_in_db: true\n" + " disable_spend_logs: false\n" + " proxy_batch_write_at: 1\n" + "router_settings:\n" + " disable_cooldowns: true\n" + ) + with wire_server(respond) as wire: + with owned_proxy( + gateway, + tmp_path, + {"FAL_AI_QUEUE_API_BASE": wire.url, "FAL_AI_API_KEY": "synthetic-fal-key"}, + config=config, + ) as candidate: + submit: Final = candidate.request( + "POST", + "/fal_ai/fal-ai/moondream3-preview/query", + {"image_url": "https://example.com/in.png", "prompt": "one word"}, + ) + assert submit.status_code == 400, submit.text + assert "pricing" in submit.text + assert wire.drain() == () + + +@pytest.mark.covers("other.provider_wire.fal_ai.passthrough_queue_submit_prices_string_resolution_like_integer") +def test_fal_queue_submit_prices_string_resolution_like_the_integer(gateway: Gateway, tmp_path: Path) -> None: + def respond(request: Request) -> Reply: + assert request.headers["authorization"] == "Key synthetic-fal-key" + return Reply(body=json.dumps({"request_id": "req-1", "status": "IN_QUEUE"}).encode()) + + numeric_body: Final = {"image_url": "https://example.com/in.png", "resolution": 512} + string_body: Final = {"image_url": "https://example.com/in.png", "resolution": "512"} + config: Final = tmp_path / "proxy_config.yaml" + config.write_text( + "model_list: []\n" + "general_settings:\n" + " master_key: os.environ/LITELLM_MASTER_KEY\n" + " database_url: os.environ/DATABASE_URL\n" + " store_model_in_db: true\n" + " disable_spend_logs: false\n" + " proxy_batch_write_at: 1\n" + "router_settings:\n" + " disable_cooldowns: true\n" + ) + with wire_server(respond) as wire: + with owned_proxy( + gateway, + tmp_path, + {"FAL_AI_QUEUE_API_BASE": wire.url, "FAL_AI_API_KEY": "synthetic-fal-key"}, + config=config, + ) as candidate: + numeric: Final = candidate.request("POST", f"/fal_ai/{_MODEL}", numeric_body) + assert numeric.status_code == 200, numeric.text + assert json.loads(numeric.content) == {"request_id": "req-1", "status": "IN_QUEUE"} + string: Final = candidate.request("POST", f"/fal_ai/{_MODEL}", string_body) + assert string.status_code == 200, string.text + assert json.loads(string.content) == {"request_id": "req-1", "status": "IN_QUEUE"} + numeric_rows: Final = eventually( + lambda: read_rows( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (numeric.headers["x-litellm-call-id"],), + ), + lambda values: len(values) == 1, + seconds=70, + ) + string_rows: Final = eventually( + lambda: read_rows( + 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (string.headers["x-litellm-call-id"],), + ), + lambda values: len(values) == 1, + seconds=70, + ) + numeric_spend_value: Final = numeric_rows[0]["spend"] + string_spend_value: Final = string_rows[0]["spend"] + assert isinstance(numeric_spend_value, (int, float)) + assert isinstance(string_spend_value, (int, float)) + numeric_spend: Final = float(numeric_spend_value) + string_spend: Final = float(string_spend_value) + assert numeric_spend > 0, f"resolution 512 logged {numeric_spend} spend" + assert string_spend > 0, f'resolution "512" logged {string_spend} spend' + assert numeric_spend == string_spend, ( + f'resolution 512 was billed {numeric_spend} but resolution "512" was billed {string_spend}' + ) + forwarded: Final = wire.drain() + assert [(request.method, request.target) for request in forwarded] == [ + ("POST", f"/{_MODEL}"), + ("POST", f"/{_MODEL}"), + ] + assert [json.loads(request.body) for request in forwarded] == [numeric_body, string_body] diff --git a/tests/integration/providers/test_fal_ai_video_wire.py b/tests/integration/providers/test_fal_ai_video_wire.py index 827818c6780..90df4520b98 100644 --- a/tests/integration/providers/test_fal_ai_video_wire.py +++ b/tests/integration/providers/test_fal_ai_video_wire.py @@ -1,14 +1,58 @@ +import datetime +import ipaddress import json +import ssl import uuid +from pathlib import Path from typing import Final import pytest -from integration._support.client import Gateway +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID +from integration._support.client import JSON_OBJECT, Gateway, object_value, string_value from integration._support.wire import Reply, Request, wire_server _MODEL: Final = "bytedance/seedance-2.5/text-to-video" _H3_MODEL: Final = "minimax/h3/text-to-video" _MP4: Final = b"\x00\x00\x00\x18ftypmp42" + uuid.uuid4().bytes * 4 +_OVERSIZED_SIDE: Final = "9" * 30 + + +def _h3_queue_reply(request_id: str) -> Reply: + return Reply(body=json.dumps({"status": "IN_QUEUE", "request_id": request_id, "queue_position": 0}).encode()) + + +def _write_self_signed_cert(cert_dir: Path) -> tuple[Path, Path]: + key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + now: Final = datetime.datetime.now(datetime.timezone.utc) + name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")]) + cert: Final = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=7)) + .add_extension( + x509.SubjectAlternativeName([x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), + critical=False, + ) + .sign(key, hashes.SHA256()) + ) + cert_file: Final = cert_dir / "cert.pem" + key_file: Final = cert_dir / "key.pem" + cert_file.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_file.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.NoEncryption(), + ) + ) + return cert_file, key_file @pytest.mark.covers("other.provider_wire.fal_ai.video_queue_create_status_and_content_download") @@ -76,6 +120,9 @@ def test_fal_h3_video_create_uses_canonical_body_and_status_path(gateway: Gatewa request_id: Final = "fal-h3-req-" + uuid.uuid4().hex def respond(request: Request) -> Reply: + if request.target == f"/files/{request_id}.mp4": + assert request.method == "GET" + return Reply(body=_MP4, content_type="video/mp4") assert request.headers["authorization"] == "Key synthetic-fal-key" if request.method == "POST": assert request.target == f"/{_H3_MODEL}" @@ -170,7 +217,217 @@ def test_fal_video_failed_result_reports_failed_status_and_fal_error(gateway: Ga video_id: Final = created["id"] status: Final = gateway.get(f"/v1/videos/{video_id}") assert status["status"] == "failed" - assert "input.reference_image_urls: Failed to download the file" in status["error"]["message"] + assert "input.reference_image_urls: Failed to download the file" in string_value( + object_value(status["error"])["message"] + ) content: Final = gateway.request("GET", f"/v1/videos/{video_id}/content") assert content.status_code == 422, content.text assert "Failed to download the file" in content.text + + +@pytest.mark.covers("other.provider_wire.fal_ai.video_result_probe_forwards_extra_headers") +def test_fal_result_probe_carries_the_deployment_extra_headers(gateway: Gateway) -> None: + request_id: Final = "fal-probe-req-" + uuid.uuid4().hex + marker: Final = uuid.uuid4().hex + + def respond(request: Request) -> Reply: + assert request.headers["authorization"] == "Key synthetic-fal-key" + if request.method == "POST": + assert request.target == f"/{_H3_MODEL}" + return Reply( + body=json.dumps({"status": "IN_QUEUE", "request_id": request_id, "queue_position": 0}).encode() + ) + assert request.method == "GET" + if request.target == f"/minimax/h3/requests/{request_id}/status": + return Reply(body=json.dumps({"status": "COMPLETED", "request_id": request_id}).encode()) + assert request.target == f"/minimax/h3/requests/{request_id}" + if request.headers.get("x-integration-routing") != marker: + return Reply(status=403, body=json.dumps({"detail": "routing header missing"}).encode()) + return Reply(body=json.dumps({"video": {"url": f"{wire_url}/files/{request_id}.mp4"}}).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + wire_url: Final = wire.url + model: Final = scenario.model( + model=f"fal_ai/{_H3_MODEL}", + api_base=wire.url, + api_key="synthetic-fal-key", + extra_headers={"x-integration-routing": marker}, + ) + created: Final = gateway.post( + "/v1/videos", + { + "model": model, + "prompt": "a paper boat drifting across a puddle after rain", + "seconds": 6, + "size": "2k", + }, + ) + assert created["status"] == "queued" + video_id: Final = created["id"] + response: Final = gateway.request("GET", f"/v1/videos/{video_id}") + assert response.status_code == 200, response.text + status: Final = JSON_OBJECT.validate_json(response.content) + assert status["status"] == "completed", status + assert status["error"] is None, status + assert [ + (request.method, request.target, request.headers.get("x-integration-routing")) for request in wire.drain() + ] == [ + ("POST", f"/{_H3_MODEL}", marker), + ("GET", f"/minimax/h3/requests/{request_id}/status", marker), + ("GET", f"/minimax/h3/requests/{request_id}", marker), + ] + + +@pytest.mark.covers("other.provider_wire.fal_ai.video_result_probe_honors_ssl_verify") +def test_fal_result_probe_reuses_the_ssl_verify_false_client(gateway: Gateway, tmp_path: Path) -> None: + request_id: Final = "fal-tls-req-" + uuid.uuid4().hex + cert_file, key_file = _write_self_signed_cert(tmp_path) + context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(certfile=cert_file, keyfile=key_file) + + def respond(request: Request) -> Reply: + assert request.headers["authorization"] == "Key synthetic-fal-key" + if request.method == "POST": + assert request.target == f"/{_H3_MODEL}" + return Reply( + body=json.dumps({"status": "IN_QUEUE", "request_id": request_id, "queue_position": 0}).encode() + ) + assert request.method == "GET" + if request.target == f"/minimax/h3/requests/{request_id}/status": + return Reply(body=json.dumps({"status": "COMPLETED", "request_id": request_id}).encode()) + assert request.target == f"/minimax/h3/requests/{request_id}" + return Reply(body=json.dumps({"video": {"url": f"{wire_url}/files/{request_id}.mp4"}}).encode()) + + with wire_server(respond, tls=context) as wire, gateway.scenario() as scenario: + wire_url: Final = wire.url + model: Final = scenario.model( + model=f"fal_ai/{_H3_MODEL}", + api_base=wire.url, + api_key="synthetic-fal-key", + ssl_verify=False, + ) + created: Final = gateway.post( + "/v1/videos", + { + "model": model, + "prompt": "a paper boat drifting across a puddle after rain", + "seconds": 6, + "size": "2k", + }, + ) + assert created["status"] == "queued" + video_id: Final = created["id"] + response: Final = gateway.request("GET", f"/v1/videos/{video_id}") + assert response.status_code == 200, response.text + status: Final = JSON_OBJECT.validate_json(response.content) + assert status["status"] == "completed", status + assert status["error"] is None, status + assert [(request.method, request.target) for request in wire.drain()] == [ + ("POST", f"/{_H3_MODEL}"), + ("GET", f"/minimax/h3/requests/{request_id}/status"), + ("GET", f"/minimax/h3/requests/{request_id}"), + ] + + +@pytest.mark.covers("other.provider_wire.fal_ai.video_result_probe_hangup_stays_completed") +def test_fal_provider_hanging_up_on_the_result_probe_keeps_the_completed_status(gateway: Gateway) -> None: + request_id: Final = "fal-hangup-req-" + uuid.uuid4().hex + + def respond(request: Request) -> Reply: + assert request.headers["authorization"] == "Key synthetic-fal-key" + if request.method == "POST": + assert request.target == f"/{_H3_MODEL}" + return Reply( + body=json.dumps({"status": "IN_QUEUE", "request_id": request_id, "queue_position": 0}).encode() + ) + assert request.method == "GET" + if request.target == f"/minimax/h3/requests/{request_id}/status": + return Reply(body=json.dumps({"status": "COMPLETED", "request_id": request_id}).encode()) + assert request.target == f"/minimax/h3/requests/{request_id}" + return Reply(chunks=(b"{",), abort_after=0) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"fal_ai/{_H3_MODEL}", + api_base=wire.url, + api_key="synthetic-fal-key", + ) + created: Final = gateway.post( + "/v1/videos", + { + "model": model, + "prompt": "a paper boat drifting across a puddle after rain", + "seconds": 6, + "size": "2k", + }, + ) + assert created["status"] == "queued" + video_id: Final = created["id"] + response: Final = gateway.request("GET", f"/v1/videos/{video_id}") + assert response.status_code == 200, response.text + status: Final = JSON_OBJECT.validate_json(response.content) + assert status["status"] == "completed", status + assert status["error"] is None, status + assert [(request.method, request.target) for request in wire.drain()] == [ + ("POST", f"/{_H3_MODEL}"), + ("GET", f"/minimax/h3/requests/{request_id}/status"), + ("GET", f"/minimax/h3/requests/{request_id}"), + ] + + +@pytest.mark.covers("other.provider_wire.fal_ai.h3_auto_duration_omits_duration_and_queues") +def test_fal_h3_auto_duration_omits_duration_and_queues(gateway: Gateway) -> None: + request_id: Final = "fal-h3-auto-" + uuid.uuid4().hex + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == f"/{_H3_MODEL}" + assert json.loads(request.body) == { + "prompt": "a cat playing volleyball on a beach", + "resolution": "768P", + "aspect_ratio": "16:9", + } + return _h3_queue_reply(request_id) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"fal_ai/{_H3_MODEL}", api_base=wire.url, api_key="synthetic-fal-key") + response: Final = gateway.request( + "POST", + "/v1/videos", + {"model": model, "prompt": "a cat playing volleyball on a beach", "seconds": "auto", "size": "1280x720"}, + ) + assert response.status_code == 200, response.text + assert response.json()["status"] == "queued" + assert [(request.method, request.target) for request in wire.drain()] == [("POST", f"/{_H3_MODEL}")] + + +@pytest.mark.covers("other.provider_wire.fal_ai.h3_oversized_size_uses_top_resolution_tier") +def test_fal_h3_oversized_size_uses_top_resolution_tier_and_queues(gateway: Gateway) -> None: + request_id: Final = "fal-h3-oversized-" + uuid.uuid4().hex + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == f"/{_H3_MODEL}" + assert json.loads(request.body) == { + "prompt": "a cat playing volleyball on a beach", + "duration": 5, + "resolution": "4K", + "aspect_ratio": "1:1", + } + return _h3_queue_reply(request_id) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"fal_ai/{_H3_MODEL}", api_base=wire.url, api_key="synthetic-fal-key") + response: Final = gateway.request( + "POST", + "/v1/videos", + { + "model": model, + "prompt": "a cat playing volleyball on a beach", + "seconds": "5", + "size": f"{_OVERSIZED_SIDE}x{_OVERSIZED_SIDE}", + }, + ) + assert response.status_code == 200, response.text + assert response.json()["status"] == "queued" + assert [(request.method, request.target) for request in wire.drain()] == [("POST", f"/{_H3_MODEL}")] diff --git a/tests/integration/providers/test_xiaomi_mimo_wire.py b/tests/integration/providers/test_xiaomi_mimo_wire.py new file mode 100644 index 00000000000..96b9dc17bdf --- /dev/null +++ b/tests/integration/providers/test_xiaomi_mimo_wire.py @@ -0,0 +1,258 @@ +import json +import uuid +from collections.abc import Mapping +from pathlib import Path +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +_BACKENDS: Final = ("mimo-v2.6-pro", "mimo-v2.6-flash") +_API_KEY: Final = "synthetic-xiaomi-key" +_ARITHMETIC_PROMPT: Final = "What is 17 + 26? Answer with just the number." +_WEATHER_PROMPT: Final = "What is the weather in Paris? Use the tool." +_COUNTING_PROMPT: Final = "Count from 1 to 5, one number per line." +_WEATHER_TOOL: Final[JsonValue] = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, +} +_COST_MAP_PATH: Final = Path(__file__).resolve().parents[3] / "model_prices_and_context_window.json" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_COST_MAP: Final = TypeAdapter(dict[str, dict[str, object]]) + + +class _Delta(BaseModel): + model_config = ConfigDict(extra="ignore") + content: str | None = None + reasoning_content: str | None = None + + +class _Choice(BaseModel): + model_config = ConfigDict(extra="ignore") + delta: _Delta + finish_reason: str | None = None + + +class _Chunk(BaseModel): + model_config = ConfigDict(extra="ignore") + id: str + choices: tuple[_Choice, ...] + + +def _catalog_cost(backend: str, field: str) -> float: + cost_map: Final = _COST_MAP.validate_json(_COST_MAP_PATH.read_bytes()) + cost_value: Final = cost_map[f"xiaomi_mimo/{backend}"][field] + assert isinstance(cost_value, (int, float)) + return float(cost_value) + + +def _approx(value: float) -> object: + return pytest.approx(value, rel=1e-6) # pyright: ignore[reportUnknownMemberType] # pytest lacks typed approx stubs + + +def _completion(identity: str, backend: str, message: Mapping[str, object], finish: str) -> bytes: + return json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": backend, + "choices": [{"index": 0, "message": message, "finish_reason": finish}], + "usage": {"prompt_tokens": 23, "completion_tokens": 41, "total_tokens": 64}, + } + ).encode() + + +def _frame(identity: str, backend: str, delta: Mapping[str, object], finish: str | None = None) -> bytes: + value: Final = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": backend, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], + } + return b"data: " + json.dumps(value).encode() + b"\n\n" + + +def _assert_provider_request(request: Request, backend: str, prompt: str) -> dict[str, JsonValue]: + assert request.method == "POST" + assert request.target == "/chat/completions" + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + assert request.headers["content-type"] == "application/json" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["model"] == backend + assert body["messages"] == [{"role": "user", "content": prompt}] + return body + + +@pytest.mark.covers("other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing") +@pytest.mark.parametrize("backend", _BACKENDS) +def test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price(gateway: Gateway, backend: str) -> None: + identity: Final = f"xiaomi-cost-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + body: Final = _assert_provider_request(request, backend, _ARITHMETIC_PROMPT) + assert body["max_tokens"] == 256 + assert "max_completion_tokens" not in body + return Reply( + body=_completion( + identity, + backend, + {"role": "assistant", "content": "43", "reasoning_content": "17 plus 26 is 43."}, + "stop", + ) + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"xiaomi_mimo/{backend}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": _ARITHMETIC_PROMPT}], + "max_completion_tokens": 256, + }, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["id"] == identity + assert payload["choices"] == [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "role": "assistant", + "content": "43", + "reasoning_content": "17 plus 26 is 43.", + "provider_specific_fields": {"refusal": None}, + }, + "provider_specific_fields": {}, + } + ] + assert payload["usage"] == {"prompt_tokens": 23, "completion_tokens": 41, "total_tokens": 64} + expected_cost: Final = 23 * _catalog_cost(backend, "input_cost_per_token") + 41 * _catalog_cost( + backend, "output_cost_per_token" + ) + assert float(response.headers["x-litellm-response-cost"]) == _approx(expected_cost) + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")] + rows: Final = eventually( + lambda: read_rows( + 'SELECT spend, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (identity,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert (rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) == (23, 41) + spend: Final = rows[0]["spend"] + assert isinstance(spend, (int, float, str)) + assert float(spend) == _approx(expected_cost) + + +@pytest.mark.covers("other.provider_wire.xiaomi_mimo.reasoning_and_answer_stream_as_deltas") +def test_xiaomi_mimo_stream_delivers_reasoning_then_answer_deltas(gateway: Gateway) -> None: + backend: Final = _BACKENDS[0] + identity: Final = f"xiaomi-stream-{uuid.uuid4().hex}" + frames: Final = ( + _frame(identity, backend, {"role": "assistant", "reasoning_content": "Count "}), + _frame(identity, backend, {"reasoning_content": "up by one."}), + _frame(identity, backend, {"content": "1\n2\n"}), + _frame(identity, backend, {"content": "3\n4\n5"}), + _frame(identity, backend, {}, finish="stop"), + b"data: [DONE]\n\n", + ) + + def respond(request: Request) -> Reply: + body: Final = _assert_provider_request(request, backend, _COUNTING_PROMPT) + assert body["stream"] is True + return Reply(content_type="text/event-stream", chunks=frames) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"xiaomi_mimo/{backend}", api_base=wire.url, api_key=_API_KEY) + with gateway.client.stream( + "POST", + "/v1/chat/completions", + json={"model": model, "messages": [{"role": "user", "content": _COUNTING_PROMPT}], "stream": True}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) as response: + assert response.status_code == 200, response.read() + lines: Final = tuple(line for line in response.iter_lines() if line.startswith("data: ")) + assert lines[-1] == "data: [DONE]" + chunks: Final = tuple(_Chunk.model_validate_json(line.removeprefix("data: ")) for line in lines[:-1]) + assert {chunk.id for chunk in chunks} == {identity} + choices: Final = tuple(choice for chunk in chunks for choice in chunk.choices) + assert "".join(choice.delta.reasoning_content or "" for choice in choices) == "Count up by one." + assert "".join(choice.delta.content or "" for choice in choices) == "1\n2\n3\n4\n5" + assert tuple(choice.finish_reason for choice in choices if choice.finish_reason) == ("stop",) + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")] + + +@pytest.mark.covers("other.provider_wire.xiaomi_mimo.tool_call_survives_translation") +def test_xiaomi_mimo_tool_call_is_forwarded_and_returned(gateway: Gateway) -> None: + backend: Final = _BACKENDS[1] + identity: Final = f"xiaomi-tool-{uuid.uuid4().hex}" + tool_call: Final = { + "id": "call_paris", + "type": "function", + "function": {"name": "get_weather", "arguments": json.dumps({"city": "Paris"})}, + } + + def respond(request: Request) -> Reply: + body: Final = _assert_provider_request(request, backend, _WEATHER_PROMPT) + assert body["tools"] == [_WEATHER_TOOL] + assert body["tool_choice"] == "auto" + return Reply( + body=_completion( + identity, + backend, + { + "role": "assistant", + "content": None, + "reasoning_content": "Need the tool.", + "tool_calls": [tool_call], + }, + "tool_calls", + ) + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"xiaomi_mimo/{backend}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": _WEATHER_PROMPT}], + "tools": [_WEATHER_TOOL], + "tool_choice": "auto", + }, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["choices"] == [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "role": "assistant", + "content": None, + "reasoning_content": "Need the tool.", + "tool_calls": [tool_call], + "provider_specific_fields": {"refusal": None}, + }, + "provider_specific_fields": {}, + } + ] + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")] diff --git a/tests/llm_translation/conftest.py b/tests/llm_translation/conftest.py index 567040c1d19..a88dcf4ae3e 100644 --- a/tests/llm_translation/conftest.py +++ b/tests/llm_translation/conftest.py @@ -7,6 +7,8 @@ import asyncio import importlib +from collections.abc import Generator +from typing import Final import pytest @@ -20,6 +22,7 @@ from tests._vcr_conftest_common import ( # noqa: E402,F401 emit_cassette_cache_session_banner, emit_vcr_classification_summary, emit_vcr_diagnostic_log, + guard_vcr_patch_points, install_live_call_probe, record_vcr_outcome, register_persister_if_enabled, @@ -37,17 +40,12 @@ def fake_openai_endpoint(): # Per-item respx detection (``apply_vcr_auto_marker_to_items``) handles # the vast majority of respx-vs-vcrpy conflicts automatically. The entries -# below are the persister's and the WebSocket VCR's own unit-test files, which -# exercise ``save_cassette`` / ``load_cassette`` against fakeredis and must not -# themselves run under a live cassette context. +# below are the persister's, the WebSocket VCR's, and the cassette patch-leak +# guard's own unit-test files, which exercise ``save_cassette`` / +# ``load_cassette`` against fakeredis or enter cassettes themselves and must +# not run under a live cassette context. _VCR_AUTO_MARKER_SKIP_FILES = frozenset( - {"test_vcr_redis_persister.py", "test_ws_vcr.py"} -) - -_VCR_INCOMPATIBLE_NODEID_SUFFIXES: tuple[str, ...] = ( - "test_nvidia_nim.py::test_embedding_nvidia_nim", - "test_litellm_proxy_provider.py::test_litellm_gateway_from_sdk_embedding[False]", - "test_litellm_proxy_provider.py::test_litellm_gateway_from_sdk_embedding[True]", + {"test_vcr_redis_persister.py", "test_ws_vcr.py", "test_vcr_leak_guard.py"} ) @@ -77,6 +75,17 @@ def _vcr_outcome_gate(request, vcr): record_vcr_outcome(request, vcr) +@pytest.hookimpl(wrapper=True, trylast=True) +def pytest_runtest_teardown(item: pytest.Item) -> Generator[None, object, object]: + try: + result: Final = yield + except BaseException: + guard_vcr_patch_points(item, teardown_failed=True) + raise + guard_vcr_patch_points(item, teardown_failed=False) + return result + + def pytest_configure(config): _verbose_state.remember_pluginmanager(config) reset_vcr_diag_dir() @@ -172,7 +181,6 @@ def pytest_collection_modifyitems(config, items): apply_vcr_auto_marker_to_items( items, skip_files=_VCR_AUTO_MARKER_SKIP_FILES, - skip_nodeid_suffixes=_VCR_INCOMPATIBLE_NODEID_SUFFIXES, ) custom_logger_tests = [ diff --git a/tests/llm_translation/test_litellm_proxy_provider.py b/tests/llm_translation/test_litellm_proxy_provider.py index 8630259877d..a10fc55ecc5 100644 --- a/tests/llm_translation/test_litellm_proxy_provider.py +++ b/tests/llm_translation/test_litellm_proxy_provider.py @@ -2,6 +2,8 @@ import json import re from datetime import datetime from io import BytesIO +from pathlib import Path +from typing import Final from unittest.mock import AsyncMock @@ -12,7 +14,12 @@ import pytest from unittest.mock import MagicMock, patch from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler import pytest_asyncio -from openai import AsyncOpenAI +from openai import AsyncOpenAI, OpenAI +from openai.types import CreateEmbeddingResponse, Embedding +from openai.types.create_embedding_response import Usage + +from tests.capturing_transport import CapturingTransport +from tests._vcr_conftest_common import rewound_new_episodes_cassette @pytest.mark.asyncio @@ -87,62 +94,61 @@ async def test_litellm_gateway_from_sdk_structured_output(): assert "json_schema" in json_schema -@pytest.mark.parametrize("is_async", [False, True]) +_GATEWAY_EMBEDDING_RESPONSE: Final = CreateEmbeddingResponse( + object="list", + data=(Embedding(object="embedding", index=0, embedding=(0.1, 0.2, 0.3)),), + model="my-vllm-model", + usage=Usage(prompt_tokens=2, total_tokens=2), +) + + +async def _gateway_embedding_via_injected_client( + is_async: bool, +) -> tuple[CapturingTransport, litellm.EmbeddingResponse]: + transport: Final = CapturingTransport(_GATEWAY_EMBEDDING_RESPONSE) + response: Final = ( + await litellm.aembedding( + model="litellm_proxy/my-vllm-model", + input="Hello world", + client=AsyncOpenAI(api_key="fake-key", http_client=httpx.AsyncClient(transport=transport)), + api_base="my-custom-api-base", + ) + if is_async + else litellm.embedding( + model="litellm_proxy/my-vllm-model", + input="Hello world", + client=OpenAI(api_key="fake-key", http_client=httpx.Client(transport=transport)), + api_base="my-custom-api-base", + ) + ) + return transport, response + + +@pytest.mark.parametrize("is_async", (False, True)) @pytest.mark.asyncio -async def test_litellm_gateway_from_sdk_embedding(is_async): +async def test_litellm_gateway_from_sdk_embedding(is_async: bool): litellm.set_verbose = True litellm._turn_on_debug() - captured_bodies = [] - - def handler(request: httpx.Request) -> httpx.Response: - captured_bodies.append(json.loads(request.content)) - return httpx.Response( - 200, - json={ - "object": "list", - "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], - "model": "my-vllm-model", - "usage": {"prompt_tokens": 2, "total_tokens": 2}, - }, - ) - - if is_async: - from openai import AsyncOpenAI - - openai_client = AsyncOpenAI( - api_key="fake-key", - http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), - ) - response = await litellm.aembedding( - model="litellm_proxy/my-vllm-model", - input="Hello world", - client=openai_client, - api_base="my-custom-api-base", - ) - else: - from openai import OpenAI - - openai_client = OpenAI( - api_key="fake-key", - http_client=httpx.Client(transport=httpx.MockTransport(handler)), - ) - response = litellm.embedding( - model="litellm_proxy/my-vllm-model", - input="Hello world", - client=openai_client, - api_base="my-custom-api-base", - ) - - request_body = captured_bodies[0] - print("Request body - {}".format(request_body)) + transport, response = await _gateway_embedding_via_injected_client(is_async) + request_body: Final = transport.request_bodies[0] assert "Hello world" == request_body["input"] assert "my-vllm-model" == request_body["model"] assert "encoding_format" not in request_body assert response.data[0]["embedding"] == [0.1, 0.2, 0.3] +@pytest.mark.asyncio +async def test_litellm_gateway_from_sdk_embedding_under_foreign_cassette(tmp_path: Path): + with rewound_new_episodes_cassette(tmp_path): + sync_transport, _ = await _gateway_embedding_via_injected_client(is_async=False) + async_transport, _ = await _gateway_embedding_via_injected_client(is_async=True) + + assert tuple(body["input"] for body in sync_transport.request_bodies) == ("Hello world",) + assert tuple(body["input"] for body in async_transport.request_bodies) == ("Hello world",) + + @pytest.mark.parametrize("is_async", [False, True]) @pytest.mark.asyncio async def test_litellm_gateway_from_sdk_image_generation(is_async): diff --git a/tests/llm_translation/test_nvidia_nim.py b/tests/llm_translation/test_nvidia_nim.py index d5942e674d0..0f16c01fd2f 100644 --- a/tests/llm_translation/test_nvidia_nim.py +++ b/tests/llm_translation/test_nvidia_nim.py @@ -1,17 +1,21 @@ import json from datetime import datetime +from typing import Final from unittest.mock import AsyncMock import httpx import pytest +from openai.types import CreateEmbeddingResponse, Embedding +from openai.types.create_embedding_response import Usage as EmbeddingUsage from unittest.mock import patch, MagicMock import litellm from litellm import Choices, Message, ModelResponse, EmbeddingResponse, Usage from litellm import completion from base_rerank_unit_tests import BaseLLMRerankTest +from tests.capturing_transport import CapturingTransport def test_completion_nvidia_nim(): @@ -63,33 +67,23 @@ def test_embedding_nvidia_nim(): litellm.set_verbose = True from openai import OpenAI - captured_bodies = [] - - def handler(request: httpx.Request) -> httpx.Response: - captured_bodies.append(json.loads(request.content)) - return httpx.Response( - 200, - json={ - "object": "list", - "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], - "model": "nvidia/nv-embedqa-e5-v5", - "usage": {"prompt_tokens": 6, "total_tokens": 6}, - }, + transport: Final = CapturingTransport( + CreateEmbeddingResponse( + object="list", + data=(Embedding(object="embedding", index=0, embedding=(0.1, 0.2, 0.3)),), + model="nvidia/nv-embedqa-e5-v5", + usage=EmbeddingUsage(prompt_tokens=6, total_tokens=6), ) - - client = OpenAI( - api_key="fake-api-key", - http_client=httpx.Client(transport=httpx.MockTransport(handler)), ) - response = litellm.embedding( + client: Final = OpenAI(api_key="fake-api-key", http_client=httpx.Client(transport=transport)) + response: Final = litellm.embedding( model="nvidia_nim/nvidia/nv-embedqa-e5-v5", input="What is the meaning of life?", input_type="passage", dimensions=1024, client=client, ) - request_body = captured_bodies[0] - print("request_body: ", request_body) + request_body: Final = transport.request_bodies[0] assert request_body["input"] == "What is the meaning of life?" assert request_body["model"] == "nvidia/nv-embedqa-e5-v5" assert request_body["input_type"] == "passage" diff --git a/tests/llm_translation/test_vcr_leak_guard.py b/tests/llm_translation/test_vcr_leak_guard.py new file mode 100644 index 00000000000..5372342790b --- /dev/null +++ b/tests/llm_translation/test_vcr_leak_guard.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import re +from pathlib import Path +from typing import Final + +import httpx +import httpx2 +import pytest + +from tests._vcr_conftest_common import ( + detect_vcr_patch_leak, + guard_vcr_patch_points, + restore_vcr_patch_points, + rewound_new_episodes_cassette, +) + +_ORIGINAL_MOCK_HANDLE_ASYNC_REQUEST: Final = httpx.MockTransport.handle_async_request +_ORIGINAL_HTTPX2_MOCK_HANDLE_ASYNC_REQUEST: Final = httpx2.MockTransport.handle_async_request + + +@pytest.fixture +def leaked_cassette_dir(tmp_path: Path): + context: Final = rewound_new_episodes_cassette(tmp_path) + context.__enter__() + yield tmp_path + context.__exit__(None, None, None) + + +def test_no_leak_when_no_cassette_is_active(): + assert detect_vcr_patch_leak() is None + + +def test_leaked_cassette_is_detected_named_and_restorable(leaked_cassette_dir: Path): + leak: Final = detect_vcr_patch_leak() + + assert leak is not None + assert {"httpx.MockTransport.handle_async_request", "aiohttp.client.ClientSession._request"} <= set( + leak.patch_points + ) + assert leak.cassette_paths == (str(leaked_cassette_dir / "rewound_owner.yaml"),) + + restore_vcr_patch_points() + + assert detect_vcr_patch_leak() is None + assert httpx.MockTransport.handle_async_request is _ORIGINAL_MOCK_HANDLE_ASYNC_REQUEST + + +def test_leak_is_detected_on_every_transport_family_vcrpy_patches(leaked_cassette_dir: Path): + leak: Final = detect_vcr_patch_leak() + + assert leak is not None + assert "httpx2.MockTransport.handle_async_request" in leak.patch_points + assert httpx2.MockTransport.handle_async_request is not _ORIGINAL_HTTPX2_MOCK_HANDLE_ASYNC_REQUEST + + restore_vcr_patch_points() + + assert httpx2.MockTransport.handle_async_request is _ORIGINAL_HTTPX2_MOCK_HANDLE_ASYNC_REQUEST + + +def test_guard_fails_the_leaking_test_and_restores_the_originals(request, leaked_cassette_dir: Path): + with pytest.raises(pytest.fail.Exception, match=re.escape(request.node.nodeid)) as failure: + guard_vcr_patch_points(request.node, teardown_failed=False) + + assert str(leaked_cassette_dir / "rewound_owner.yaml") in str(failure.value) + assert detect_vcr_patch_leak() is None + + +def test_guard_restores_silently_when_the_teardown_already_failed(request, leaked_cassette_dir: Path): + guard_vcr_patch_points(request.node, teardown_failed=True) + + assert detect_vcr_patch_leak() is None diff --git a/tests/local_testing/conftest.py b/tests/local_testing/conftest.py index 228457f4d55..d03f074f557 100644 --- a/tests/local_testing/conftest.py +++ b/tests/local_testing/conftest.py @@ -93,7 +93,6 @@ _VCR_INCOMPATIBLE_FILES = frozenset( # carry no real provider cost. _VCR_INCOMPATIBLE_NODEID_SUFFIXES: tuple[str, ...] = ( "test_router.py::test_router_text_completion_client", - "test_embedding.py::test_encoding_format_omitted_by_default_for_openai_sdk", ) diff --git a/tests/local_testing/test_embedding.py b/tests/local_testing/test_embedding.py index c119334da6f..acbc4f20405 100644 --- a/tests/local_testing/test_embedding.py +++ b/tests/local_testing/test_embedding.py @@ -15,6 +15,9 @@ from unittest.mock import AsyncMock, MagicMock, patch import litellm from litellm import completion, completion_cost, embedding +from openai.types import CreateEmbeddingResponse +from openai.types.create_embedding_response import Usage as EmbeddingUsage +from tests.capturing_transport import CapturingTransport from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE litellm.set_verbose = False @@ -1268,23 +1271,15 @@ def test_encoding_format_omitted_by_default_for_openai_sdk(monkeypatch): Optional global override: `LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT`. """ monkeypatch.delenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", raising=False) - captured_bodies = [] - - def handler(request: httpx.Request) -> httpx.Response: - captured_bodies.append(json.loads(request.content)) - return httpx.Response( - 200, - json={ - "object": "list", - "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], - "model": "text-embedding-ada-002", - "usage": {"prompt_tokens": 1, "total_tokens": 1}, - }, + transport = CapturingTransport( + CreateEmbeddingResponse( + object="list", + data=(Embedding(object="embedding", index=0, embedding=(0.1, 0.2, 0.3)),), + model="text-embedding-ada-002", + usage=EmbeddingUsage(prompt_tokens=1, total_tokens=1), ) - - client = openai.OpenAI( - api_key="sk-test", http_client=httpx.Client(transport=httpx.MockTransport(handler)) ) + client = openai.OpenAI(api_key="sk-test", http_client=httpx.Client(transport=transport)) response = embedding( model="text-embedding-ada-002", @@ -1294,7 +1289,7 @@ def test_encoding_format_omitted_by_default_for_openai_sdk(monkeypatch): ) assert response.data[0]["embedding"] == [0.1, 0.2, 0.3] - assert "encoding_format" not in captured_bodies[0], ( + assert "encoding_format" not in transport.request_bodies[0], ( "encoding_format should be omitted from the upstream request when not provided by user" ) diff --git a/tests/logging_callback_tests/test_langsmith_unit_test.py b/tests/logging_callback_tests/test_langsmith_unit_test.py index 17cd63d8974..341f71f3b4a 100644 --- a/tests/logging_callback_tests/test_langsmith_unit_test.py +++ b/tests/logging_callback_tests/test_langsmith_unit_test.py @@ -356,10 +356,12 @@ async def test_langsmith_key_based_logging(): # tenant_id should not be in headers if not provided assert "x-tenant-id" not in call_args[1]["headers"] + assert call_args[1]["headers"]["Content-Type"] == "application/json" + # Verify the request body contains the expected data - request_body = call_args[1]["json"] + request_body = json.loads(call_args[1]["content"]) assert "post" in request_body - assert len(request_body["post"]) == 1 # Should contain one run + assert len(request_body["post"]) == 1 # EXPECTED BODY expected_body = { @@ -404,7 +406,7 @@ async def test_langsmith_key_based_logging(): } # Print both bodies for debugging - actual_body = call_args[1]["json"] + actual_body = json.loads(call_args[1]["content"]) print("\nExpected body:") print(json.dumps(expected_body, indent=2)) print("\nActual body:") diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index 2b92367f186..dfd250338ff 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -456,9 +456,15 @@ async def test_sse_mcp_handler_mock(): """Test the SSE MCP handler functionality""" from litellm.proxy._types import UserAPIKeyAuth - # Mock the SSE session manager and its methods - mock_sse_session_manager = AsyncMock() - mock_sse_session_manager.handle_request = AsyncMock() + read_stream, write_stream = MagicMock(), MagicMock() + + @asynccontextmanager + async def connect_sse(scope, receive, send): + yield read_stream, write_stream + + mock_sse = MagicMock() + mock_sse.connect_sse.side_effect = connect_sse + run = AsyncMock() # Mock scope, receive, send with proper ASGI scope format mock_scope = { @@ -483,13 +489,14 @@ async def test_sse_mcp_handler_mock(): ) with ( + patch("litellm.proxy._experimental.mcp_server.server.server.run", run), patch( "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), patch( - "litellm.proxy._experimental.mcp_server.server.sse_session_manager", - mock_sse_session_manager, + "litellm.proxy._experimental.mcp_server.server.sse", + mock_sse, ), patch( "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", @@ -504,10 +511,8 @@ async def test_sse_mcp_handler_mock(): # Call the handler await handle_sse_mcp(mock_scope, mock_receive, mock_send) - # Verify SSE session manager handle_request was called - mock_sse_session_manager.handle_request.assert_called_once_with( - mock_scope, mock_receive, mock_send - ) + assert run.await_args.args[:2] == (read_stream, write_stream) + assert mock_sse.connect_sse.call_args.args[0]["path"] == "/mcp/sse" @pytest.mark.asyncio @@ -545,7 +550,10 @@ async def test_sse_mcp_handler_propagates_passthrough_401(): True, ), patch( - "litellm.proxy._experimental.mcp_server.server.sse_session_manager", + "litellm.proxy._experimental.mcp_server.server.sse", + ) as transport, + patch( + "litellm.proxy._experimental.mcp_server.server.server.run", AsyncMock(), ), patch( @@ -569,6 +577,7 @@ async def test_sse_mcp_handler_propagates_passthrough_401(): with pytest.raises(HTTPException) as excinfo: await handle_sse_mcp(mock_scope, mock_receive, mock_send) + transport.connect_sse.assert_not_called() assert excinfo.value.status_code == 401 assert excinfo.value.headers and "WWW-Authenticate" in excinfo.value.headers @@ -2953,7 +2962,7 @@ async def test_call_mcp_tool_uses_manager_permission_lookup(): mcp_info={"server_name": "test_server"}, ) - expected_response = [TextContent(type="text", text="ok")] + expected_response = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) with ( patch.object( @@ -3029,7 +3038,7 @@ async def test_call_mcp_tool_resolves_unprefixed_tool_name_and_checks_permission mcp_info={"server_name": "test_server"}, ) - expected_response = [TextContent(type="text", text="ok")] + expected_response = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) with ( patch.object( diff --git a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py index 1d4e13474a7..6c57e59f7e3 100644 --- a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py +++ b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py @@ -413,6 +413,7 @@ PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES = { "/comprehendmedical/{operation}": {"POST"}, "/transcribe": {"POST"}, "/transcribe/{operation}": {"POST"}, + "/tinyfish/{endpoint:path}": {"GET", "POST"}, } diff --git a/tests/pass_through_unit_tests/test_vertex_ai_anthropic_streaming_cost_injection.py b/tests/pass_through_unit_tests/test_vertex_ai_anthropic_streaming_cost_injection.py index 498f0a734a3..ab5fd7f80ee 100644 --- a/tests/pass_through_unit_tests/test_vertex_ai_anthropic_streaming_cost_injection.py +++ b/tests/pass_through_unit_tests/test_vertex_ai_anthropic_streaming_cost_injection.py @@ -53,6 +53,7 @@ async def test_vertex_ai_anthropic_streaming_cost_injection_enabled(): # Setup logging object with model info litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj) + litellm_logging_obj.litellm_params = {} litellm_logging_obj.model_call_details = {"model": "claude-sonnet-4@20250514"} litellm_logging_obj.completion_start_time = None litellm_logging_obj.async_success_handler = AsyncMock() @@ -132,6 +133,7 @@ async def test_vertex_ai_anthropic_streaming_cost_injection_disabled(): response.aiter_bytes = mock_aiter_bytes litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj) + litellm_logging_obj.litellm_params = {} litellm_logging_obj.model_call_details = {"model": "claude-sonnet-4@20250514"} litellm_logging_obj.completion_start_time = None litellm_logging_obj.async_success_handler = AsyncMock() @@ -194,6 +196,7 @@ async def test_vertex_ai_anthropic_streaming_cost_injection_no_usage_chunk(): response.aiter_bytes = mock_aiter_bytes litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj) + litellm_logging_obj.litellm_params = {} litellm_logging_obj.model_call_details = {"model": "claude-sonnet-4@20250514"} litellm_logging_obj.completion_start_time = None litellm_logging_obj.async_success_handler = AsyncMock() @@ -249,6 +252,7 @@ async def test_vertex_ai_anthropic_streaming_model_extraction(): response.aiter_bytes = mock_aiter_bytes litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj) + litellm_logging_obj.litellm_params = {} litellm_logging_obj.model_call_details = {} litellm_logging_obj.completion_start_time = None litellm_logging_obj.async_success_handler = AsyncMock() diff --git a/tests/proxy_migration_tests/test_prisma_toolchain.py b/tests/proxy_migration_tests/test_prisma_toolchain.py index 4e2274cc582..556c680a84a 100644 --- a/tests/proxy_migration_tests/test_prisma_toolchain.py +++ b/tests/proxy_migration_tests/test_prisma_toolchain.py @@ -25,7 +25,6 @@ from collections.abc import Callable from pathlib import Path import pytest - from litellm_proxy_extras.prisma_toolchain import ( DEFAULT_PRISMA_COMMAND_TIMEOUT, DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT, @@ -36,14 +35,16 @@ from litellm_proxy_extras.prisma_toolchain import ( heal_incomplete_nodeenv_cache, node_binary_path, prisma_bootstrap_timeout, - prisma_command_timeout, prisma_cli_available, + prisma_command_timeout, prisma_migrate_deploy_timeout, resolve_prisma_argv, run_prisma, ) from litellm_proxy_extras.utils import ProxyExtrasDBManager +from tests._process_helpers import process_is_gone + REPO_ROOT = Path(__file__).resolve().parents[2] PROXY_EXTRAS = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras" @@ -280,17 +281,6 @@ def test_migrate_deploy_stops_at_its_own_timeout( assert elapsed < 30 -def _process_is_gone(pid: int, within_seconds: float) -> bool: - deadline = time.monotonic() + within_seconds - while time.monotonic() < deadline: - try: - os.kill(pid, 0) - except ProcessLookupError: - return True - time.sleep(0.05) - return False - - def test_a_timed_out_migrate_deploy_takes_its_process_tree_with_it( toolchain_env: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -309,7 +299,7 @@ def test_a_timed_out_migrate_deploy_takes_its_process_tree_with_it( grandchild_pid = int(pidfile.read_text()) try: assert len(_deploy_calls(log_path)) == 2 - assert _process_is_gone(grandchild_pid, within_seconds=5) + assert process_is_gone(grandchild_pid, within_seconds=5) finally: try: os.kill(grandchild_pid, signal.SIGKILL) diff --git a/tests/proxy_unit_tests/test_custom_tokenizer_bug.py b/tests/proxy_unit_tests/test_custom_tokenizer_bug.py index c4b1f4f3afd..23b6b302202 100644 --- a/tests/proxy_unit_tests/test_custom_tokenizer_bug.py +++ b/tests/proxy_unit_tests/test_custom_tokenizer_bug.py @@ -22,10 +22,8 @@ from litellm.proxy.proxy_server import token_counter def _fake_hf_tokenizer(num_tokens: int) -> MagicMock: - encoding = MagicMock() - encoding.__len__.return_value = num_tokens tokenizer = MagicMock() - tokenizer.encode_batch_fast.return_value = [encoding] + tokenizer.encode_batch_fast.return_value = [[0] * num_tokens] return tokenizer @@ -58,7 +56,7 @@ async def test_custom_tokenizer_from_model_info_is_used(monkeypatch): ) monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) - with patch.object(litellm.utils, "Tokenizer") as mock_tokenizer_cls: + with patch.object(litellm.utils, "tokenizer_dispatch") as mock_tokenizer_cls: mock_tokenizer_cls.from_pretrained.return_value = _fake_hf_tokenizer(7) response = await token_counter( @@ -92,7 +90,7 @@ async def test_model_without_custom_tokenizer_uses_default(monkeypatch): ) monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", llm_router) - with patch.object(litellm.utils, "Tokenizer") as mock_tokenizer_cls: + with patch.object(litellm.utils, "tokenizer_dispatch") as mock_tokenizer_cls: response = await token_counter( request=TokenCountRequest( model="gpt-4", diff --git a/tests/proxy_unit_tests/test_jwt_key_mapping.py b/tests/proxy_unit_tests/test_jwt_key_mapping.py index 3f2c04336a7..e95ed42013b 100644 --- a/tests/proxy_unit_tests/test_jwt_key_mapping.py +++ b/tests/proxy_unit_tests/test_jwt_key_mapping.py @@ -19,6 +19,8 @@ from litellm.proxy._types import ( ) from litellm.proxy.management_endpoints.jwt_key_mapping_endpoints import ( _to_response, + _token_hash_for_create, + _token_hash_for_update, create_jwt_key_mapping, delete_jwt_key_mapping, info_jwt_key_mapping, @@ -1515,6 +1517,132 @@ def test_jwt_client_id_field_does_not_raise_on_duplicate(): assert auth.virtual_key_claim_field == "new_field" +# ────────────────────────────────────────────── +# Tests: identifying the mapped key by hash instead of plaintext +# ────────────────────────────────────────────── + +_TOKEN_HASH = "1923314ae0efc8b2523c7d421bac5a7cf88df291273b139948b526d396974a41" + + +def test_create_stores_a_supplied_token_hash_verbatim(): + """A caller that holds only the hash gets it stored as given, not hashed again.""" + from litellm.proxy._types import CreateJWTKeyMappingRequest + + data = CreateJWTKeyMappingRequest( + jwt_claim_name="email", jwt_claim_value="user@example.com", token=_TOKEN_HASH + ) + + assert _token_hash_for_create(data) == _TOKEN_HASH + + +def test_create_hashes_a_supplied_plaintext_key(): + """Supplying `key` keeps the original behaviour, so existing configs are unaffected.""" + from litellm.proxy._types import CreateJWTKeyMappingRequest, hash_token + + data = CreateJWTKeyMappingRequest( + jwt_claim_name="email", jwt_claim_value="user@example.com", key="sk-test-key" + ) + + assert _token_hash_for_create(data) == hash_token("sk-test-key") + + +@pytest.mark.parametrize( + "kwargs", + [ + pytest.param({}, id="neither"), + pytest.param({"key": "sk-test-key", "token": _TOKEN_HASH}, id="both"), + ], +) +def test_create_requires_exactly_one_identifier(kwargs): + """Neither or both is a 400, so a mapping can never be created ambiguously.""" + from litellm.proxy._types import CreateJWTKeyMappingRequest + + data = CreateJWTKeyMappingRequest( + jwt_claim_name="email", jwt_claim_value="user@example.com", **kwargs + ) + + with pytest.raises(HTTPException) as exc_info: + _token_hash_for_create(data) + + assert exc_info.value.status_code == 400 + assert "exactly one" in exc_info.value.detail.lower() + + +@pytest.mark.parametrize( + "token", + [ + pytest.param("sk-not-a-hash", id="plaintext-key"), + pytest.param("abc123", id="too-short"), + pytest.param(_TOKEN_HASH.upper(), id="uppercase"), + pytest.param(_TOKEN_HASH + "0", id="too-long"), + pytest.param(_TOKEN_HASH[:-1] + "g", id="non-hex-character"), + ], +) +def test_create_rejects_a_token_that_is_not_a_sha256_hash(token): + """hash_token hashes unconditionally, so a bad `token` would be stored as a hash + of a hash and then silently match nothing at auth time.""" + from litellm.proxy._types import CreateJWTKeyMappingRequest + + data = CreateJWTKeyMappingRequest( + jwt_claim_name="email", jwt_claim_value="user@example.com", token=token + ) + + with pytest.raises(HTTPException) as exc_info: + _token_hash_for_create(data) + + assert exc_info.value.status_code == 400 + assert "SHA-256" in exc_info.value.detail + + +def test_update_leaves_the_mapped_key_alone_when_neither_is_given(): + """Updating only the description must not blank out the mapped key.""" + from litellm.proxy._types import UpdateJWTKeyMappingRequest + + data = UpdateJWTKeyMappingRequest(id="mapping-1", description="new text") + + assert _token_hash_for_update(data) is None + + +def test_update_stores_a_supplied_token_hash_verbatim(): + from litellm.proxy._types import UpdateJWTKeyMappingRequest + + data = UpdateJWTKeyMappingRequest(id="mapping-1", token=_TOKEN_HASH) + + assert _token_hash_for_update(data) == _TOKEN_HASH + + +def test_update_hashes_a_supplied_plaintext_key(): + from litellm.proxy._types import UpdateJWTKeyMappingRequest, hash_token + + data = UpdateJWTKeyMappingRequest(id="mapping-1", key="sk-rotated") + + assert _token_hash_for_update(data) == hash_token("sk-rotated") + + +def test_update_rejects_both_identifiers(): + from litellm.proxy._types import UpdateJWTKeyMappingRequest + + data = UpdateJWTKeyMappingRequest(id="mapping-1", key="sk-abc", token=_TOKEN_HASH) + + with pytest.raises(HTTPException) as exc_info: + _token_hash_for_update(data) + + assert exc_info.value.status_code == 400 + assert "at most one" in exc_info.value.detail.lower() + + +def test_update_rejects_a_token_that_is_not_a_sha256_hash(): + from litellm.proxy._types import UpdateJWTKeyMappingRequest + + data = UpdateJWTKeyMappingRequest(id="mapping-1", token="sk-not-a-hash") + + with pytest.raises(HTTPException) as exc_info: + _token_hash_for_update(data) + + assert exc_info.value.status_code == 400 + assert "SHA-256" in exc_info.value.detail + + # ────────────────────────────────────────────── # Tests: cache eviction must happen AFTER the DB write commits # ────────────────────────────────────────────── diff --git a/tests/proxy_unit_tests/test_user_api_key_auth.py b/tests/proxy_unit_tests/test_user_api_key_auth.py index f5e8d861d79..9cdac341b1f 100644 --- a/tests/proxy_unit_tests/test_user_api_key_auth.py +++ b/tests/proxy_unit_tests/test_user_api_key_auth.py @@ -1540,18 +1540,17 @@ async def test_user_budget_lookup_is_also_unenforced_when_the_database_is_down() """ KNOWN LIMITATION, pinned deliberately rather than discovered later. - `get_user_object` cannot tell "row absent" from "database unreachable": the - absent case raises inside its own try (auth_checks.py:2177) and the handler - at :2213 rewrites every exception into the same - `ValueError("User doesn't exist in db...")`. A connection error, a query - timeout and a malformed row all reach us as that one type and message. + `get_user_object` lets a connection-level outage propagate as-is and rewrites + every other read failure (a query-level Prisma error, a malformed row) into + the same `ValueError("User doesn't exist in db...")` as an absent row, and + `_read_user_model_max_budget` swallows every exception either way. - So tolerating the absent case, which the test above requires, unavoidably - tolerates an outage too, and a user who DOES have a per-model budget goes + So tolerating the absent case, which the test above requires, also + tolerates an outage, and a user who DOES have a per-model budget goes unenforced while the DB is unreachable. This is pre-existing behaviour of - `get_user_object` that the virtual-key path inherits identically; it is not - introduced here. Distinguishing them needs a dedicated exception type for - the absent case and a change to both auth paths. + the virtual-key path; it is not introduced here. Distinguishing them needs + `_read_user_model_max_budget` to let an outage through the way the JWT + path does. """ from litellm.caching.dual_cache import DualCache from litellm.proxy.auth.user_api_key_auth import _read_user_model_max_budget diff --git a/tests/test_litellm/caching/test_caching.py b/tests/test_litellm/caching/test_caching.py index c7ec8abc31e..2e4122530d8 100644 --- a/tests/test_litellm/caching/test_caching.py +++ b/tests/test_litellm/caching/test_caching.py @@ -1,3 +1,4 @@ +import asyncio import logging import re from unittest.mock import MagicMock @@ -6,8 +7,9 @@ import pytest import litellm.caching.redis_cache as redis_cache_module from litellm.caching.caching import Cache +from litellm.caching.caching_handler import _PENDING_CACHE_WRITES from litellm.caching.redis_cache import RedisCache, _RedisTimeoutLogThrottle -from litellm.types.caching import LiteLLMCacheType, SemanticCacheScope +from litellm.types.caching import EMBEDDING_CACHE_FORMAT_VERSION, LiteLLMCacheType, SemanticCacheScope from litellm.types.utils import Embedding, EmbeddingResponse, Usage @@ -278,3 +280,112 @@ def test_exact_cache_key_includes_anthropic_messages_params(anthropic_param): assert baseline != cache.get_cache_key( model="claude-sonnet-4-5", messages=messages, **anthropic_param ) + + +@pytest.mark.asyncio +async def test_embedding_cache_skips_write_when_one_input_yields_many_embeddings(monkeypatch): + """A cross-encoder behind /embeddings returns one score per document for a single + input string; caching data[0] per input would make the second call return 1 score.""" + import litellm + from litellm import CustomLLM + + class ScoreEveryDocument(CustomLLM): + provider_calls: int = 0 + + async def aembedding(self, model, input, model_response, **kwargs) -> EmbeddingResponse: + self.provider_calls += 1 + return EmbeddingResponse( + model=model, + data=[Embedding(embedding=[float(i)], index=i, object="embedding") for i in range(5)], + ) + + scorer = ScoreEveryDocument() + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "score-every-doc", "custom_handler": scorer}]) + monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "score-every-doc"]) + monkeypatch.setattr(litellm, "_custom_providers", [*litellm._custom_providers, "score-every-doc"]) + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL)) + + batch = '{"query": "q", "documents": ["a", "b", "c", "d", "e"]}' + first = await litellm.aembedding(model="score-every-doc/m", input=[batch]) + await asyncio.gather(*_PENDING_CACHE_WRITES) + second = await litellm.aembedding(model="score-every-doc/m", input=[batch]) + + assert scorer.provider_calls == 2 + assert [len(first.data), len(second.data)] == [5, 5] + + +@pytest.mark.asyncio +async def test_embedding_cache_refetches_entries_written_without_format_version(monkeypatch): + import litellm + from litellm import CustomLLM + + class EmbedLength(CustomLLM): + provider_calls: int = 0 + + async def aembedding(self, model, input, model_response, **kwargs) -> EmbeddingResponse: + self.provider_calls += 1 + return EmbeddingResponse( + model=model, + data=[ + Embedding(embedding=[float(len(text))], index=idx, object="embedding") + for idx, text in enumerate(input) + ], + ) + + embedder = EmbedLength() + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "embed-length", "custom_handler": embedder}]) + monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "embed-length"]) + monkeypatch.setattr(litellm, "_custom_providers", [*litellm._custom_providers, "embed-length"]) + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL)) + + await litellm.aembedding(model="embed-length/m", input=["abcd"]) + await asyncio.gather(*_PENDING_CACHE_WRITES) + store = litellm.cache.cache.cache_dict + stored = [entry["response"] for entry in store.values()] + assert [entry["format_version"] for entry in stored] == [EMBEDDING_CACHE_FORMAT_VERSION], stored + legacy_store = { + key: { + **entry, + "response": { + field: value + for field, value in {**entry["response"], "embedding": [-1.0]}.items() + if field != "format_version" + }, + } + for key, entry in store.items() + } + monkeypatch.setattr(litellm.cache.cache, "cache_dict", legacy_store) + + refetched = await litellm.aembedding(model="embed-length/m", input=["abcd"]) + + assert embedder.provider_calls == 2, "an entry written without format_version must be a cache miss" + assert [item["embedding"] for item in refetched.data] == [[4.0]] + + +@pytest.mark.asyncio +async def test_embedding_cache_serves_base64_string_embeddings_on_repeat(monkeypatch): + import litellm + from litellm import CustomLLM + + class Base64Embedder(CustomLLM): + provider_calls: int = 0 + + async def aembedding(self, model, input, model_response, **kwargs) -> EmbeddingResponse: + self.provider_calls += 1 + return EmbeddingResponse( + model=model, + data=[Embedding(embedding="AACAPwAAAEA=", index=idx, object="embedding") for idx, _ in enumerate(input)], + ) + + embedder = Base64Embedder() + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "embed-b64", "custom_handler": embedder}]) + monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "embed-b64"]) + monkeypatch.setattr(litellm, "_custom_providers", [*litellm._custom_providers, "embed-b64"]) + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL)) + + first = await litellm.aembedding(model="embed-b64/m", input=["abcd"]) + await asyncio.gather(*_PENDING_CACHE_WRITES) + second = await litellm.aembedding(model="embed-b64/m", input=["abcd"]) + + assert embedder.provider_calls == 1, "a string embedding written to the cache must be served on repeat" + assert [item["embedding"] for item in second.data] == [item["embedding"] for item in first.data] == ["AACAPwAAAEA="] diff --git a/tests/test_litellm/caching/test_caching_handler.py b/tests/test_litellm/caching/test_caching_handler.py index 29266464a21..d8027da5afa 100644 --- a/tests/test_litellm/caching/test_caching_handler.py +++ b/tests/test_litellm/caching/test_caching_handler.py @@ -14,7 +14,7 @@ from fastapi.testclient import TestClient from datetime import datetime from unittest.mock import AsyncMock -from litellm.caching.caching_handler import LLMCachingHandler +from litellm.caching.caching_handler import _PENDING_CACHE_WRITES, LLMCachingHandler @pytest.mark.asyncio @@ -833,3 +833,46 @@ async def test_agentic_loop_followup_cache_hit_with_converted_stream_marker_repl assert hit.cached_result.choices[0].message.content == "done" logging_obj.handle_sync_success_callbacks_for_async_calls.assert_called_once() assert logging_obj.handle_sync_success_callbacks_for_async_calls.call_args.kwargs["cache_hit"] is True + + +@pytest.mark.asyncio +async def test_partial_embedding_cache_hit_sends_only_misses_and_keeps_input_order(monkeypatch): + import litellm + from litellm import CustomLLM + from litellm.caching.caching import Cache + from litellm.types.utils import Embedding, EmbeddingResponse + + class RecordingEmbedder(CustomLLM): + provider_inputs: tuple[tuple[str, ...], ...] = () + + async def aembedding(self, model, input, model_response, **kwargs) -> EmbeddingResponse: + self.provider_inputs = (*self.provider_inputs, tuple(input)) + return EmbeddingResponse( + model=model, + data=[ + Embedding(embedding=[float(len(text))], index=idx, object="embedding") + for idx, text in enumerate(input) + ], + ) + + embedder = RecordingEmbedder() + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "recording-embedder", "custom_handler": embedder}]) + monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list, "recording-embedder"]) + monkeypatch.setattr(litellm, "_custom_providers", [*litellm._custom_providers, "recording-embedder"]) + monkeypatch.setattr(litellm, "cache", Cache(type="local")) + + await litellm.aembedding(model="recording-embedder/m", input=["aa", "bbbb"]) + await asyncio.gather(*_PENDING_CACHE_WRITES) + mixed_input = ["c", "aa", "ddd", "bbbb", "eeeee"] + response = await litellm.aembedding(model="recording-embedder/m", input=mixed_input) + await asyncio.gather(*_PENDING_CACHE_WRITES) + + assert embedder.provider_inputs == (("aa", "bbbb"), ("c", "ddd", "eeeee")), embedder.provider_inputs + assert [item["index"] for item in response.data] == [0, 1, 2, 3, 4] + assert [item["embedding"] for item in response.data] == [[float(len(text))] for text in mixed_input] + assert response._hidden_params["cache_hit"] is True, "a partial hit must still be reported as a cache hit" + + repeat = await litellm.aembedding(model="recording-embedder/m", input=mixed_input) + + assert len(embedder.provider_inputs) == 2, embedder.provider_inputs + assert [item["embedding"] for item in repeat.data] == [[float(len(text))] for text in mixed_input] diff --git a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py index 6c20ef135ba..b260d240f29 100644 --- a/tests/test_litellm/experimental_mcp_client/test_mcp_client.py +++ b/tests/test_litellm/experimental_mcp_client/test_mcp_client.py @@ -552,6 +552,50 @@ class TestExecuteSessionOperationSurfacesTransportError: with pytest.raises(asyncio.CancelledError): await client._execute_session_operation(transport_ctx, _op) + @pytest.mark.asyncio + @pytest.mark.parametrize("failure_phase", ("early", "late", "mixed")) + @patch("litellm.experimental_mcp_client.client.ClientSession") + async def test_response_close_preserves_cancellation_and_original_errors(self, session_class, failure_phase): + closed: Final = asyncio.Event() + close_error: Final = httpx2.ReadError("response close failed") + connect_error: Final = httpx2.ConnectError("another request failed before cancellation") + cancelled: Final = asyncio.CancelledError("caller cancelled") + + class FailingCloseStream(httpx2.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b"pending" + + async def aclose(self) -> None: + closed.set() + raise close_error + + client: Final = MCPClient(server_url="https://example.com/mcp") + async with client._create_httpx_client_factory( + transport=httpx2.MockTransport(lambda _: httpx2.Response(200, stream=FailingCloseStream())) + )() as http_client: + response: Final = await http_client.send(http_client.build_request("POST", client.server_url), stream=True) + + async def initialize(): + if failure_phase == "early": + await response.aclose() + raise cancelled + + async def close_transport(*args): + if failure_phase == "early": + return + try: + await response.aclose() + except httpx2.ReadError as error: + failures: Final = [error, connect_error] if failure_phase == "mixed" else [error] + raise _FakeExceptionGroup("transport", [_FakeExceptionGroup("reader", failures)]) + + self._make_session(session_class, initialize) + expected: Final = close_error if failure_phase == "early" else connect_error if failure_phase == "mixed" else cancelled + with pytest.raises(type(expected)) as caught: + await client._execute_session_operation(self._make_transport(close_transport), AsyncMock(), http_client) + assert caught.value is expected + assert closed.is_set() + @pytest.mark.asyncio @patch("litellm.experimental_mcp_client.client.ClientSession") async def test_cleanup_error_after_success_is_swallowed(self, mock_session_cls): @@ -568,6 +612,94 @@ class TestExecuteSessionOperationSurfacesTransportError: assert result == "done" + @pytest.mark.asyncio + @patch("litellm.experimental_mcp_client.client.ClientSession") + async def test_session_entry_failure_still_closes_transport(self, session_class): + failure: Final = RuntimeError("session dispatcher did not start") + session_class.return_value.__aenter__ = AsyncMock(side_effect=failure) + closed: Final = asyncio.Event() + + async def close_transport(*args): + await anyio.lowlevel.checkpoint() + closed.set() + + transport: Final = self._make_transport(close_transport) + client: Final = MCPClient(server_url="https://example.com/mcp") + with pytest.raises(RuntimeError) as caught: + await client._execute_session_operation(transport, AsyncMock()) + assert caught.value is failure + assert closed.is_set() + + @pytest.mark.asyncio + @pytest.mark.parametrize("original_error", (False, True)) + @patch("litellm.experimental_mcp_client.client.ClientSession") + async def test_session_exit_cancellation_preserves_original_failure(self, session_class, original_error): + self._make_session(session_class, AsyncMock(return_value=None)) + cancelled: Final = asyncio.CancelledError("cancelled while closing session") + session_class.return_value.__aexit__ = AsyncMock(side_effect=cancelled) + original: Final = RuntimeError("operation failed") + transport: Final = self._make_transport(None) + client: Final = MCPClient(server_url="https://example.com/mcp") + + async def operation(session): + if original_error: + raise original + return "done" + + with pytest.raises(RuntimeError if original_error else asyncio.CancelledError) as caught: + await client._execute_session_operation(transport, operation) + assert caught.value is (original if original_error else cancelled) + transport.__aexit__.assert_awaited_once() + + @pytest.mark.asyncio + @pytest.mark.parametrize("signal_type", (SystemExit, KeyboardInterrupt)) + @pytest.mark.parametrize("phase", ("session", "transport")) + @patch("litellm.experimental_mcp_client.client.ClientSession") + async def test_cleanup_preserves_process_exit(self, session_class, phase, signal_type): + self._make_session(session_class, AsyncMock(return_value=None)) + signal: Final = signal_type("process stopping") + if phase == "session": + session_class.return_value.__aexit__ = AsyncMock(side_effect=signal) + transport: Final = self._make_transport(signal if phase == "transport" else None) + client: Final = MCPClient(server_url="https://example.com/mcp") + with pytest.raises(signal_type) as caught: + await client._execute_session_operation(transport, AsyncMock(return_value="done")) + assert caught.value is signal + transport.__aexit__.assert_awaited_once() + + @pytest.mark.asyncio + @patch("litellm.experimental_mcp_client.client.ClientSession") + async def test_session_and_termination_share_one_cleanup_deadline(self, session_class): + self._make_session(session_class, AsyncMock(return_value=None)) + deleting: Final = asyncio.Event() + + async def close_session(*args): + await anyio.sleep(1) + + async def respond(request: httpx2.Request) -> httpx2.Response: + deleting.set() + await anyio.sleep_forever() + raise AssertionError("termination unexpectedly resumed") + + client: Final = MCPClient(server_url="https://example.com/mcp") + http_client: Final = client._create_httpx_client_factory(transport=httpx2.MockTransport(respond))() + session_class.return_value.__aexit__ = AsyncMock(side_effect=close_session) + + async def close_transport(*args): + await http_client.delete(client.server_url) + + before: Final = anyio.current_time() + try: + with pytest.raises(asyncio.CancelledError): + await client._execute_session_operation( + self._make_transport(close_transport), AsyncMock(return_value="completed"), http_client=http_client + ) + assert deleting.is_set() + assert 4.8 <= anyio.current_time() - before < 5.8 + finally: + await http_client.aclose() + + class TestMCPClientResolvedAuth: """A pre-resolved httpx2.Auth is attached to the upstream client's auth= slot.""" @@ -736,7 +868,7 @@ async def test_run_with_session_quiet_on_error_demotes_warning_to_debug(): async def _op(_session): raise boom - async def _fake_exec(_transport_ctx, _operation): + async def _fake_exec(_transport_ctx, _operation, http_client=None): raise boom with patch.object(client, "_create_transport_context", return_value=(object(), None)): @@ -1486,7 +1618,9 @@ async def test_http_response_handler_preserves_success_and_http_errors(status_co client: Final = MCPClient(server_url="https://example.com/mcp", timeout=30) async with client._create_httpx_client_factory(transport=httpx2.MockTransport(respond))() as http_client: operation: Final = client._execute_session_operation( - streamable_http_client(client.server_url, http_client=http_client), lambda session: session.list_tools() + streamable_http_client(client.server_url, http_client=http_client), + lambda session: session.list_tools(), + http_client=http_client, ) if status_code == 200: result: Final = await asyncio.wait_for(operation, timeout=3) @@ -1816,13 +1950,14 @@ async def test_interrupted_http_response_preserves_the_transport_failure() -> No def respond(request: httpx2.Request) -> httpx2.Response: return httpx2.Response(200, headers={"Content-Type": "application/json"}, stream=_InterruptedHTTPBody()) - async with httpx2.AsyncClient(transport=httpx2.MockTransport(respond)) as http_client: - client: Final = MCPClient(server_url="https://example.com/mcp", timeout=30) + client: Final = MCPClient(server_url="https://example.com/mcp", timeout=30) + async with client._create_httpx_client_factory(transport=httpx2.MockTransport(respond))() as http_client: with pytest.raises(httpx2.RemoteProtocolError, match="secret-incomplete-response"): await asyncio.wait_for( client._execute_session_operation( streamable_http_client(client.server_url, http_client=http_client), lambda session: session.list_tools(), + http_client=http_client, ), timeout=3, ) @@ -1906,7 +2041,7 @@ async def test_optional_discovery_capabilities_and_errors( "jsonrpc": "2.0", "id": payload.id, "result": { - "protocolVersion": payload.params["protocolVersion"], + "protocolVersion": (payload.params or {})["protocolVersion"], "capabilities": {} if outcome == "absent" else {advertised if outcome == "other_capability" else capability: {}}, @@ -1985,7 +2120,7 @@ async def test_optional_discovery_uses_each_sessions_capabilities(supports_first return httpx2.Response(202) result: Final = ( { - "protocolVersion": payload.params["protocolVersion"], + "protocolVersion": (payload.params or {})["protocolVersion"], "capabilities": next(capabilities), "serverInfo": {"name": "changing", "version": "1"}, } @@ -2030,7 +2165,7 @@ async def test_optional_discovery_preserves_cancellation(method: str) -> None: "jsonrpc": "2.0", "id": payload.id, "result": { - "protocolVersion": payload.params["protocolVersion"], + "protocolVersion": (payload.params or {})["protocolVersion"], "capabilities": {"resources": {}, "prompts": {}}, "serverInfo": {"name": "pending", "version": "1"}, }, @@ -2112,7 +2247,7 @@ async def test_optional_discovery_collects_all_pages(method: str, session_id: st "jsonrpc": "2.0", "id": payload.id, "result": { - "protocolVersion": payload.params["protocolVersion"], + "protocolVersion": (payload.params or {})["protocolVersion"], "capabilities": {"prompts": {}, "resources": {}}, "serverInfo": {"name": "paged", "version": "1"}, }, @@ -2200,7 +2335,7 @@ async def test_optional_discovery_rejects_incomplete_walks( "jsonrpc": "2.0", "id": payload.id, "result": { - "protocolVersion": payload.params["protocolVersion"], + "protocolVersion": (payload.params or {})["protocolVersion"], "capabilities": {"prompts": {}, "resources": {}}, "serverInfo": {"name": "interrupted", "version": "1"}, }, @@ -2281,7 +2416,7 @@ async def test_optional_discovery_allows_exhaustion_at_page_cap(method: str, mon return httpx2.Response(202) if payload.method == "initialize": result: Final = { - "protocolVersion": payload.params["protocolVersion"], + "protocolVersion": (payload.params or {})["protocolVersion"], "capabilities": {"prompts": {}, "resources": {}}, "serverInfo": {"name": "empty-pages", "version": "1"}, } @@ -2455,3 +2590,309 @@ def test_public_mcp_import_preserves_incompatible_sdk_error() -> None: assert not isinstance(caught.value, ModuleNotFoundError) assert caught.value.__cause__ is None assert "litellm[mcp]" not in str(caught.value) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("grouped", (False, True)) +@pytest.mark.parametrize("raise_on_error", (False, True)) +@pytest.mark.parametrize("termination", ("ok", "failure", "hang")) +async def test_outer_deadline_delivers_session_termination(termination: str, grouped: bool, raise_on_error: bool) -> None: + deleted: Final = asyncio.Event() + started: Final = asyncio.Event() + + async def respond(request: httpx2.Request) -> httpx2.Response: + await anyio.lowlevel.checkpoint() + if request.method == "DELETE": + first_termination: Final = not deleted.is_set() + deleted.set() + if termination == "hang" and first_termination: + await anyio.sleep_forever() + return httpx2.Response(500 if termination == "failure" else 200) + if request.method == "GET": + return httpx2.Response(405) + payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content) + if not isinstance(payload, JSONRPCRequest): + return httpx2.Response(202) + if payload.method == "initialize": + return httpx2.Response( + 200, + headers={"mcp-session-id": "cancel-owned-session"}, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "result": { + "protocolVersion": "2025-11-25", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "cancellation-peer", "version": "1"}, + }, + }, + ) + if payload.method == "tools/list": + return httpx2.Response(200, json={"jsonrpc": "2.0", "id": payload.id, "result": {"tools": []}}) + started.set() + await anyio.sleep_forever() + raise AssertionError("cancelled request resumed") + + client: Final = _MockTransportClient(respond, server_url="https://example.com/mcp", timeout=30) + + async def invoke(): + with anyio.fail_after(0.2): + pending: Final = client.call_tool(CallToolRequestParams(name="slow", arguments={}), raise_on_error=raise_on_error) + if grouped: + await asyncio.gather(pending) + else: + await pending + + before: Final = anyio.current_time() + with pytest.raises(TimeoutError): + await invoke() + assert started.is_set() + assert deleted.is_set(), "Cancellation must deliver DELETE before returning to the caller" + + assert anyio.current_time() - before < 6.5 + assert await client.list_tools(raise_on_error=True) == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("original_error", (False, True)) +async def test_task_cancellation_during_cleanup_preserves_failure(original_error: bool) -> None: + deleting: Final = asyncio.Event() + drained: Final = asyncio.Event() + original: Final = RuntimeError("operation failed before teardown") + + async def respond(request: httpx2.Request) -> httpx2.Response: + if request.method == "DELETE": + deleting.set() + try: + await anyio.sleep_forever() + finally: + drained.set() + if request.method == "GET": + return httpx2.Response(405) + payload: Final = _JSONRPC_MESSAGE_ADAPTER.validate_json(request.content) + if not isinstance(payload, JSONRPCRequest): + return httpx2.Response(202) + return httpx2.Response( + 200, + headers={"mcp-session-id": "cleanup-session"}, + json={ + "jsonrpc": "2.0", + "id": payload.id, + "result": { + "protocolVersion": (payload.params or {})["protocolVersion"], + "capabilities": {}, + "serverInfo": {"name": "cleanup-peer", "version": "1"}, + }, + }, + ) + + async def operation(session: mcp_client_module.ClientSession) -> str: + if original_error: + raise original + return "completed" + + client: Final = _MockTransportClient(respond, server_url="https://example.com/mcp", timeout=30) + task: Final = asyncio.create_task(client.run_with_session(operation)) + await asyncio.wait_for(deleting.wait(), 2) + for _ in range(3): + task.cancel() + await asyncio.sleep(0) + with pytest.raises(RuntimeError if original_error else asyncio.CancelledError) as caught: + await task + assert drained.is_set(), "Caller must wait for termination cleanup to finish" + if original_error: + assert caught.value is original + else: + assert task.cancelled() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("original_error", (False, True)) +@pytest.mark.parametrize("cancel_mode", ("task", "scope")) +async def test_http_close_cancellation_cannot_turn_into_success(original_error: bool, cancel_mode: str) -> None: + closing: Final = asyncio.Event() + drained: Final = asyncio.Event() + original: Final = RuntimeError("failed before HTTP close") + + class ClosingHTTPClient(httpx2.AsyncClient): + async def aclose(self) -> None: + closing.set() + try: + await anyio.sleep_forever() + finally: + drained.set() + + class ClosingMCPClient(MCPClient): + def _create_transport_context(self): + http_client: Final = ClosingHTTPClient(transport=httpx2.MockTransport(lambda _: httpx2.Response(200))) + return streamable_http_client(self.server_url, http_client=http_client), http_client + + async def _execute_session_operation(self, transport_ctx, operation, http_client=None): + if original_error: + raise original + return "completed" + + client: Final = ClosingMCPClient(server_url="https://example.com/mcp") + + async def invoke() -> str: + with anyio.fail_after(0.05 if cancel_mode == "scope" else None): + return await client.run_with_session(AsyncMock()) + + task: Final = asyncio.create_task(invoke()) + await asyncio.wait_for(closing.wait(), 2) + if cancel_mode == "task": + task.cancel() + cancellation_type: Final = asyncio.CancelledError if cancel_mode == "task" else TimeoutError + with pytest.raises(RuntimeError if original_error else cancellation_type) as caught: + await task + assert drained.is_set(), "Caller must wait for HTTP closure to finish" + if original_error: + assert caught.value is original + elif cancel_mode == "task": + assert task.cancelled() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cancel_mode", ("scope", "task", "wait_for", "read_timeout")) +@pytest.mark.parametrize("concurrency", (1, 5)) +@pytest.mark.parametrize("termination", ("ok", "hang", "hang_body")) +@pytest.mark.parametrize("raise_on_error", (False, True)) +async def test_cancellation_delivers_termination_over_tcp( + cancel_mode: str, concurrency: int, termination: str, raise_on_error: bool +) -> None: + started: Final = asyncio.Event() + terminations: Final[list[bytes]] = [] + starts: Final[list[bytes]] = [] + stop: Final = asyncio.Event() + connections: Final[list[asyncio.Task[None]]] = [] + + async def handle_connection(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + connection: Final = asyncio.current_task() + assert connection is not None + connections.append(connection) + try: + request_line: Final = await reader.readline() + if not request_line: + return + method: Final = request_line.split()[0] + headers: Final = await reader.readuntil(b"\r\n\r\n") + length: Final = next( + ( + int(line.split(b":", 1)[1]) + for line in headers.splitlines() + if line.lower().startswith(b"content-length:") + ), + 0, + ) + try: + body: Final = await reader.readexactly(length) + except asyncio.IncompleteReadError: + return + if method == b"DELETE": + terminations.append(body) + if termination != "ok": + await stop.wait() + return + writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + elif method == b"GET": + writer.write(b"HTTP/1.1 405 Method Not Allowed\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + else: + payload: Final = json.loads(body) + if payload["method"] == "tools/call": + if termination == "hang_body": + writer.write( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" + b"Content-Length: 200\r\nConnection: close\r\n\r\n" + ) + await writer.drain() + starts.append(body) + if len(starts) == concurrency: + started.set() + await stop.wait() + return + if payload["method"] == "initialize": + response: Final = json.dumps( + { + "jsonrpc": "2.0", + "id": payload["id"], + "result": { + "protocolVersion": "2025-06-18", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "tcp-peer", "version": "1"}, + }, + } + ).encode() + writer.write( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nMcp-Session-Id: tcp-session\r\n" + + f"Content-Length: {len(response)}\r\nConnection: close\r\n\r\n".encode() + + response + ) + else: + writer.write(b"HTTP/1.1 202 Accepted\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + await writer.drain() + finally: + writer.close() + await writer.wait_closed() + + listener: Final = await asyncio.start_server(handle_connection, "127.0.0.1", 0) + port: Final = listener.sockets[0].getsockname()[1] + client: Final = MCPClient( + server_url=f"http://127.0.0.1:{port}/mcp", timeout=2 if cancel_mode == "read_timeout" else 0.5 if termination != "ok" else 30 + ) + + async def calls(): + results: Final = await asyncio.gather( + *( + client.call_tool(CallToolRequestParams(name="slow", arguments={}), raise_on_error=raise_on_error) + for _ in range(concurrency) + ), + return_exceptions=cancel_mode == "read_timeout", + ) + if cancel_mode == "read_timeout": + if raise_on_error: + assert all(isinstance(result, TimeoutError) for result in results) + else: + assert all(isinstance(result, CallToolResult) and result.is_error for result in results) + return results + + async def invoke(): + if cancel_mode == "scope": + with anyio.fail_after(0.2): + return await calls() + return await calls() + + try: + task: Final = asyncio.create_task(invoke()) + await asyncio.wait_for(started.wait(), 3) + if cancel_mode == "task": + task.cancel() + expected_error: Final = ( + TimeoutError + if cancel_mode == "read_timeout" + else asyncio.CancelledError + if cancel_mode == "task" + else TimeoutError + ) + if cancel_mode == "read_timeout": + done, _ = await asyncio.wait((task,), timeout=8) + assert task in done, "Read timeout and bounded cleanup must complete without external cancellation" + await task + elif cancel_mode == "wait_for": + with pytest.raises(expected_error): + await asyncio.wait_for(task, 0.2) + else: + with pytest.raises(expected_error): + await task + assert len(starts) == concurrency + assert len(terminations) == concurrency, "Each cancelled call must send DELETE over a fresh TCP connection" + finally: + stop.set() + if not task.done(): + task.cancel() + await asyncio.wait((task,), timeout=8) + listener.close() + for connection in connections: + connection.cancel() + closed: Final = await asyncio.wait_for(asyncio.gather(*connections, return_exceptions=True), 2) + assert all(result is None or isinstance(result, asyncio.CancelledError) for result in closed), closed + await asyncio.wait_for(listener.wait_closed(), 2) diff --git a/tests/test_litellm/integrations/arize/test_arize.py b/tests/test_litellm/integrations/arize/test_arize.py index cdafd856b49..5fde627680d 100644 --- a/tests/test_litellm/integrations/arize/test_arize.py +++ b/tests/test_litellm/integrations/arize/test_arize.py @@ -5,6 +5,8 @@ from unittest.mock import MagicMock, Mock, patch # Adds the grandparent directory to sys.path to allow importing project modules import asyncio +import datetime +from collections.abc import Callable import pytest from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter @@ -64,9 +66,7 @@ async def test_arize_dynamic_params(): print(f"Tracer calls: {len(tracer_calls)}") # We should have captured calls for both requests - assert ( - len(tracer_calls) >= 2 - ), f"Expected at least 2 tracer calls, got {len(tracer_calls)}" + assert len(tracer_calls) >= 2, f"Expected at least 2 tracer calls, got {len(tracer_calls)}" # Check that we have the expected dynamic params in the kwargs team1_found = False @@ -87,9 +87,7 @@ async def test_arize_dynamic_params(): assert team1_found, "team1 dynamic params not found" assert team2_found, "team2 dynamic params not found" - print( - "✅ All assertions passed - OpenTelemetry logger correctly received dynamic params" - ) + print("✅ All assertions passed - OpenTelemetry logger correctly received dynamic params") @pytest.mark.asyncio @@ -114,11 +112,8 @@ async def test_arize_dynamic_headers_in_grpc_requests(): "opentelemetry.exporter.otlp.proto.http.trace_exporter.OTLPSpanExporter", mock_otlp_http_exporter, ): - # Create ArizeLogger with HTTP configuration - config = OpenTelemetryConfig( - exporter="otlp_http", endpoint="https://otlp.arize.com/v1" - ) + config = OpenTelemetryConfig(exporter="otlp_http", endpoint="https://otlp.arize.com/v1") arize_logger = ArizeLogger(config=config) litellm.callbacks = [arize_logger] @@ -147,25 +142,17 @@ async def test_arize_dynamic_headers_in_grpc_requests(): print(f"Captured exporter headers: {exporter_headers}") # Should have multiple exporter calls (default + dynamic) - assert ( - len(exporter_headers) >= 2 - ), f"Expected at least 2 exporter calls, got {len(exporter_headers)}" + assert len(exporter_headers) >= 2, f"Expected at least 2 exporter calls, got {len(exporter_headers)}" # Find team1 and team2 headers team1_found = False team2_found = False for headers in exporter_headers: - if ( - headers.get("api_key") == "team1_api_key" - and headers.get("arize-space-id") == "team1_space_id" - ): + if headers.get("api_key") == "team1_api_key" and headers.get("arize-space-id") == "team1_space_id": team1_found = True print(f"✅ Found team1 headers: {headers}") - elif ( - headers.get("api_key") == "team2_api_key" - and headers.get("arize-space-id") == "team2_space_id" - ): + elif headers.get("api_key") == "team2_api_key" and headers.get("arize-space-id") == "team2_space_id": team2_found = True print(f"✅ Found team2 headers: {headers}") @@ -173,6 +160,119 @@ async def test_arize_dynamic_headers_in_grpc_requests(): assert team1_found, "team1 dynamic headers not found in exporter calls" assert team2_found, "team2 dynamic headers not found in exporter calls" - print( - "✅ Test passed - Dynamic Arize params correctly passed to gRPC/HTTP exporter" - ) + print("✅ Test passed - Dynamic Arize params correctly passed to gRPC/HTTP exporter") + + +_START = datetime.datetime.now() +_END = datetime.datetime.now() + + +def _sampled_arize_logger( + random_draw: Callable[[], float] | None = None, +) -> tuple[ArizeLogger, InMemorySpanExporter]: + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + logger = ArizeLogger(tracer_provider=provider, random_draw=random_draw) + return logger, exporter + + +def _request_spans(exporter: InMemorySpanExporter) -> int: + return sum(1 for span in exporter.get_finished_spans() if span.name == "litellm_request") + + +def _arize_kwargs(callback_vars: dict[str, str] | None = None) -> dict[str, object]: + kwargs: dict[str, object] = { + "model": "gpt-4", + "litellm_params": {"metadata": {}}, + "standard_logging_object": { + "id": "call-1", + "call_type": "completion", + "model": "gpt-4", + "metadata": {}, + "messages": [{"role": "user", "content": "hi"}], + }, + } + if callback_vars is not None: + kwargs["standard_callback_dynamic_params"] = callback_vars + return kwargs + + +@pytest.mark.asyncio +async def test_success_sampling_rate_zero_exports_no_spans(): + logger, exporter = _sampled_arize_logger(random_draw=lambda: 0.0) + await logger.async_log_success_event(_arize_kwargs({"arize_success_sampling_rate": "0.0"}), None, _START, _END) + assert _request_spans(exporter) == 0 + + +@pytest.mark.asyncio +async def test_success_sampling_rate_one_exports_a_span(): + logger, exporter = _sampled_arize_logger() + await logger.async_log_success_event(_arize_kwargs({"arize_success_sampling_rate": "1.0"}), None, _START, _END) + assert _request_spans(exporter) == 1 + + +@pytest.mark.asyncio +async def test_unset_sampling_rate_exports_everything(): + logger, exporter = _sampled_arize_logger() + await logger.async_log_success_event(_arize_kwargs(), None, _START, _END) + assert _request_spans(exporter) == 1 + + +@pytest.mark.asyncio +async def test_draw_above_rate_is_dropped_draw_at_rate_is_exported(): + dropped, dropped_exporter = _sampled_arize_logger(random_draw=lambda: 0.3) + await dropped.async_log_success_event(_arize_kwargs({"arize_success_sampling_rate": "0.2"}), None, _START, _END) + assert _request_spans(dropped_exporter) == 0 + + kept, kept_exporter = _sampled_arize_logger(random_draw=lambda: 0.2) + await kept.async_log_success_event(_arize_kwargs({"arize_success_sampling_rate": "0.2"}), None, _START, _END) + assert _request_spans(kept_exporter) == 1 + + +@pytest.mark.asyncio +async def test_success_and_error_rates_are_independent(): + logger, exporter = _sampled_arize_logger() + kwargs = _arize_kwargs({"arize_success_sampling_rate": "0.0", "arize_error_sampling_rate": "1.0"}) + await logger.async_log_success_event(kwargs, None, _START, _END) + await logger.async_log_failure_event(kwargs, ValueError("boom"), _START, _END) + assert _request_spans(exporter) == 1 + + logger2, exporter2 = _sampled_arize_logger() + kwargs2 = _arize_kwargs({"arize_success_sampling_rate": "1.0", "arize_error_sampling_rate": "0.0"}) + await logger2.async_log_success_event(kwargs2, None, _START, _END) + await logger2.async_log_failure_event(kwargs2, ValueError("boom"), _START, _END) + assert _request_spans(exporter2) == 1 + + +@pytest.mark.asyncio +async def test_one_draw_per_request_across_sync_and_async_handlers(): + draws: list[int] = [] + + def counting_draw() -> float: + draws.append(1) + return 0.5 + + logger, _ = _sampled_arize_logger(random_draw=counting_draw) + kwargs = _arize_kwargs({"arize_success_sampling_rate": "1.0"}) + logger.log_success_event(kwargs, None, _START, _END) + await logger.async_log_success_event(kwargs, None, _START, _END) + assert len(draws) == 1 + + +@pytest.mark.asyncio +async def test_unparsable_sampling_rate_exports_rather_than_dropping(): + logger, exporter = _sampled_arize_logger(random_draw=lambda: 0.99) + await logger.async_log_success_event(_arize_kwargs({"arize_success_sampling_rate": "abc"}), None, _START, _END) + assert _request_spans(exporter) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("bad", ["nan", "inf", "-inf", "1.5", "-0.1"]) +async def test_out_of_range_sampling_rate_exports_rather_than_dropping(bad: str): + logger, exporter = _sampled_arize_logger(random_draw=lambda: 0.99) + await logger.async_log_success_event(_arize_kwargs({"arize_success_sampling_rate": bad}), None, _START, _END) + assert _request_spans(exporter) == 1 diff --git a/tests/test_litellm/integrations/conftest.py b/tests/test_litellm/integrations/conftest.py new file mode 100644 index 00000000000..adc8e36e0af --- /dev/null +++ b/tests/test_litellm/integrations/conftest.py @@ -0,0 +1,96 @@ +import functools +import http.server +import ipaddress +import queue +import ssl +import threading +from collections.abc import Iterator +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Final + +import pytest + + +@dataclass(frozen=True, slots=True) +class TlsSink: + url: str + certificate_path: str + received: "queue.Queue[str]" + + +class _RecordingOtelHandler(http.server.BaseHTTPRequestHandler): + def __init__(self, *args: object, received: "queue.Queue[str]", **kwargs: object) -> None: + self._received: Final = received + super().__init__(*args, **kwargs) + + def do_POST(self) -> None: + length: Final = int(self.headers.get("Content-Length") or 0) + if length: + self.rfile.read(length) + self._received.put(self.path) + self.send_response(200) + self.send_header("Content-Type", "application/x-protobuf") + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, format: str, *args: object) -> None: + pass + + +def write_self_signed_cert(directory: Path, stem: str) -> tuple[Path, Path]: + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.x509.oid import NameOID + + key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")]) + certificate: Final = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.now(timezone.utc) - timedelta(minutes=1)) + .not_valid_after(datetime.now(timezone.utc) + timedelta(hours=1)) + .add_extension( + x509.SubjectAlternativeName([x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), + critical=False, + ) + .sign(key, hashes.SHA256()) + ) + certificate_path: Final = directory / f"{stem}.crt" + certificate_path.write_bytes(certificate.public_bytes(serialization.Encoding.PEM)) + key_path: Final = directory / f"{stem}.key" + key_path.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.NoEncryption(), + ) + ) + return certificate_path, key_path + + +@pytest.fixture +def tls_sink(tmp_path: Path) -> Iterator[TlsSink]: + certificate_path, key_path = write_self_signed_cert(tmp_path, "sink") + received: queue.Queue[str] = queue.Queue() + context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(str(certificate_path), str(key_path)) + server: Final = http.server.ThreadingHTTPServer( + ("127.0.0.1", 0), functools.partial(_RecordingOtelHandler, received=received) + ) + server.socket = context.wrap_socket(server.socket, server_side=True) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield TlsSink( + url=f"https://127.0.0.1:{server.server_port}", + certificate_path=str(certificate_path), + received=received, + ) + server.shutdown() + server.server_close() + thread.join(timeout=5) diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 0c95049ce05..07705e17d9a 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -2,14 +2,17 @@ baggage helpers, metrics, the typed coercion helpers, mapper branches, span-name builders, and the registry validator's failure paths. Needs the OTel SDK.""" +import contextlib import json import threading +import time from collections.abc import Iterator from contextvars import Context as ContextVarContext from dataclasses import replace from http.server import BaseHTTPRequestHandler, HTTPServer, ThreadingHTTPServer import pytest +import requests pytest.importorskip("opentelemetry") @@ -18,6 +21,9 @@ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( # noqa: ) from opentelemetry import baggage # noqa: E402 from opentelemetry.context import attach, detach # noqa: E402 +from opentelemetry._logs.severity import SeverityNumber # noqa: E402 +from opentelemetry.sdk._logs import LogData, LogRecord # noqa: E402 +from opentelemetry.sdk._logs.export import LogExportResult # noqa: E402 from opentelemetry.sdk.metrics import MeterProvider # noqa: E402 from opentelemetry.sdk.metrics.export import InMemoryMetricReader # noqa: E402 from opentelemetry.sdk.trace import TracerProvider # noqa: E402 @@ -29,11 +35,14 @@ from opentelemetry.sdk.trace.export import ( # noqa: E402 from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402 InMemorySpanExporter, ) -from opentelemetry.trace import SpanKind, get_current_span # noqa: E402 +from opentelemetry.sdk.util.instrumentation import InstrumentationScope # noqa: E402 +from opentelemetry.trace import SpanKind, TraceFlags, get_current_span # noqa: E402 from opentelemetry.trace.propagation.tracecontext import ( # noqa: E402 TraceContextTextMapPropagator, ) +import litellm # noqa: E402 +from conftest import TlsSink # noqa: E402 from litellm.integrations.otel.plumbing import context as ctx_mod # noqa: E402 from litellm.integrations.otel.plumbing import providers # noqa: E402 from litellm.integrations.otel.model.config import OpenTelemetryV2Config # noqa: E402 @@ -1281,6 +1290,50 @@ def test_operation_exception_log_event_always_carries_required_pair(): assert ExceptionEvent.STACKTRACE not in attributes +def test_operation_exception_log_event_records_without_the_events_api(): + """Recording must not import the Events API modules (removed upstream in 1.44.0); + the SDK record path still exports.""" + import importlib + import sys + from unittest.mock import patch + + from opentelemetry._logs.severity import SeverityNumber + from opentelemetry.sdk._logs.export import InMemoryLogExporter + from opentelemetry.trace import INVALID_SPAN_CONTEXT + + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + + plumbing = ("litellm.integrations.otel.plumbing.events", "litellm.integrations.otel.plumbing.providers") + without_events_api = { + **{name: module for name, module in sys.modules.items() if name not in plumbing}, + "opentelemetry._events": None, + "opentelemetry.sdk._events": None, + } + with patch.dict(sys.modules, without_events_api, clear=True): + events_mod = importlib.import_module(plumbing[0]) + providers_mod = importlib.import_module(plumbing[1]) + + log_exporter = InMemoryLogExporter() + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + logger_provider = providers_mod.build_logger_provider(cfg, log_exporter=log_exporter) + recorder = events_mod.GenAIEventRecorder(providers_mod.get_event_logger(logger_provider)) + recorder.record_operation_exception( + span_context=INVALID_SPAN_CONTEXT, + error_type="RateLimitError", + message="rate limited", + stack_trace=None, + timestamp_ns=None, + ) + + (log,) = log_exporter.get_finished_logs() + record = log.log_record + assert record.attributes[GenAIEvent.NAME_KEY] == GenAIEvent.OPERATION_EXCEPTION + assert record.attributes[ExceptionEvent.TYPE] == "RateLimitError" + assert record.attributes[ExceptionEvent.MESSAGE] == "rate limited" + assert record.severity_number == SeverityNumber.WARN + assert record.timestamp is not None + + def test_operation_exception_log_event_not_emitted_on_success(): engine, span_exporter, log_exporter = _engine_with_event_recorder() engine.emit(SpanRole.LLM_CALL, _llm_call_data(None)) @@ -1414,3 +1467,178 @@ def test_genai_mapper_guardrail_cost_in_spend_attr(): billed = dict(entry) del billed["guardrail_cost_in_spend"] assert LiteLLM.GUARDRAIL_COST_IN_SPEND not in GenAIMapper().map(GuardrailSpanData.from_logging_entry(billed)) + + +def _sampled_span_context(): + from opentelemetry.trace import SpanContext, TraceFlags, TraceState + + return SpanContext( + trace_id=0x0AF7651916CD43DD8448EB211C80319C, + span_id=0x00F067AA0BA902B7, + is_remote=False, + trace_flags=TraceFlags(TraceFlags.SAMPLED), + trace_state=TraceState(), + ) + + +def test_operation_exception_log_event_exports_through_console_exporter(): + """The emitted record serializes through a real SDK exporter: the console + exporter only handles SDK-shaped records (``to_json`` plus a resource), so + an API-shaped record crashed the export under the repo's pinned OTel.""" + import io + import json as json_mod + + from opentelemetry.sdk._logs import LoggerProvider + from opentelemetry.sdk._logs.export import ConsoleLogExporter, SimpleLogRecordProcessor + from opentelemetry.sdk.resources import Resource + + from litellm.integrations.otel.model.semconv import ExceptionEvent, GenAIEvent + from litellm.integrations.otel.plumbing.events import GenAIEventRecorder + + out = io.StringIO() + logger_provider = LoggerProvider(resource=Resource.create({"service.name": "otel-event-test"})) + logger_provider.add_log_record_processor(SimpleLogRecordProcessor(ConsoleLogExporter(out=out))) + recorder = GenAIEventRecorder(providers.get_event_logger(logger_provider), logger_provider.resource) + recorder.record_operation_exception( + span_context=_sampled_span_context(), + error_type="RateLimitError", + message="rate limited", + stack_trace=None, + timestamp_ns=None, + ) + + exported = json_mod.loads(out.getvalue()) + assert exported["attributes"][GenAIEvent.NAME_KEY] == GenAIEvent.OPERATION_EXCEPTION + assert exported["attributes"][ExceptionEvent.TYPE] == "RateLimitError" + assert exported["attributes"][ExceptionEvent.MESSAGE] == "rate limited" + assert exported["body"] == "rate limited" + assert exported["resource"]["attributes"]["service.name"] == "otel-event-test" + + +def test_operation_exception_log_event_encodes_for_otlp(): + """The OTLP log encoder reads ``log_record.resource`` and rejects a None + body on the pinned OTel line, so the event must encode into a real + ExportLogsServiceRequest, not only land in an in-memory exporter.""" + from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs + from opentelemetry.sdk._logs.export import InMemoryLogExporter + + from litellm.integrations.otel.model.semconv import GenAIEvent + from litellm.integrations.otel.plumbing.events import GenAIEventRecorder + + log_exporter = InMemoryLogExporter() + cfg = OpenTelemetryV2Config(exporter="in_memory", enable_events=True) + logger_provider = providers.build_logger_provider(cfg, log_exporter=log_exporter) + recorder = GenAIEventRecorder(providers.get_event_logger(logger_provider), logger_provider.resource) + recorder.record_operation_exception( + span_context=_sampled_span_context(), + error_type="RateLimitError", + message="rate limited", + stack_trace=None, + timestamp_ns=None, + ) + + request = encode_logs(log_exporter.get_finished_logs()) + (resource_logs,) = request.resource_logs + (scope_logs,) = resource_logs.scope_logs + (encoded,) = scope_logs.log_records + encoded_attrs = {a.key: a.value.string_value for a in encoded.attributes} + assert encoded_attrs[GenAIEvent.NAME_KEY] == GenAIEvent.OPERATION_EXCEPTION + assert encoded.body.string_value == "rate limited" + resource_attrs = {a.key: a.value.string_value for a in resource_logs.resource.attributes} + assert resource_attrs["service.name"] == logger_provider.resource.attributes["service.name"] + + + + +def _isolate_v2_otlp_tls_env(monkeypatch: pytest.MonkeyPatch) -> None: + for key in ( + "SSL_VERIFY", + "SSL_CERT_FILE", + "OTEL_EXPORTER_OTLP_CERTIFICATE", + "OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", + "OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE", + "OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE", + ): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TIMEOUT", "2") + monkeypatch.setattr(litellm, "ssl_verify", True) + + +def test_v2_otlp_http_span_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None: + _isolate_v2_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path) + cfg = OpenTelemetryV2Config(exporter="otlp_http", endpoint=tls_sink.url) + _export_one_span(cfg) + assert tls_sink.received.get(timeout=5) == "/v1/traces" + + +def test_v2_http_json_span_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None: + _isolate_v2_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path) + cfg = OpenTelemetryV2Config(exporter="http/json", endpoint=tls_sink.url) + _export_one_span(cfg) + assert tls_sink.received.get(timeout=5) == "/v1/traces" + + +def test_v2_otlp_http_metric_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None: + _isolate_v2_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path) + cfg = OpenTelemetryV2Config(exporter="otlp_http", endpoint=tls_sink.url) + reader = providers.build_metric_reader(cfg) + provider = MeterProvider(metric_readers=[reader]) + try: + provider.get_meter("v2-tls-test").create_counter("tls_export_test").add(1) + assert provider.force_flush(), "metric flush failed" + assert tls_sink.received.get(timeout=5) == "/v1/metrics" + finally: + provider.shutdown() + + +def test_v2_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None: + _isolate_v2_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path) + cfg = OpenTelemetryV2Config(exporter="otlp_http", endpoint=tls_sink.url) + exporter = providers.build_log_exporter(cfg) + try: + record = LogRecord( + timestamp=int(time.time() * 1e9), + observed_timestamp=int(time.time() * 1e9), + trace_id=0, + span_id=0, + trace_flags=TraceFlags(0), + severity_number=SeverityNumber.INFO, + body="v2-tls-test", + ) + log_data = LogData(log_record=record, instrumentation_scope=InstrumentationScope("v2-tls-test")) + result = exporter.export([log_data]) + assert result is LogExportResult.SUCCESS, f"log export failed: {result}" + assert tls_sink.received.get(timeout=5) == "/v1/logs" + finally: + exporter.shutdown() + + +def test_v2_otlp_http_export_skips_verification_when_ssl_verify_false( + monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink +) -> None: + _isolate_v2_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_VERIFY", "false") + cfg = OpenTelemetryV2Config(exporter="otlp_http", endpoint=tls_sink.url) + _export_one_span(cfg) + assert tls_sink.received.get(timeout=5) == "/v1/traces" + + +def test_v2_otlp_http_export_rejects_untrusted_collector_by_default( + monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink +) -> None: + + + _isolate_v2_otlp_tls_env(monkeypatch) + cfg = OpenTelemetryV2Config(exporter="otlp_http", endpoint=tls_sink.url) + provider = providers.build_tracer_provider(cfg) + provider.get_tracer("probe").start_span("probe").end() + try: + with contextlib.suppress(requests.exceptions.SSLError): + provider.force_flush() + assert tls_sink.received.empty(), "sink received a request it should never have trusted" + finally: + provider.shutdown() diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index 01f2a13d252..95df3709ab8 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -4,6 +4,7 @@ and the typed StandardLoggingPayload adapter. These need no OTel SDK.""" import json import logging import re +from collections.abc import Mapping from pathlib import Path from typing import Final @@ -151,9 +152,7 @@ def test_llm_call_span_name(): def _all_constants(cls): return { - getattr(cls, name) - for name in vars(cls) - if not name.startswith("__") and isinstance(getattr(cls, name), str) + getattr(cls, name) for name in vars(cls) if not name.startswith("__") and isinstance(getattr(cls, name), str) } @@ -465,9 +464,7 @@ def test_mcp_tool_call_content_gated_off_by_default(): off = MCPToolCallSpanData.from_standard_logging_payload(_mcp_payload()) assert off.arguments_json is None and off.result_json is None - on = MCPToolCallSpanData.from_standard_logging_payload( - _mcp_payload(), capture_content=True - ) + on = MCPToolCallSpanData.from_standard_logging_payload(_mcp_payload(), capture_content=True) assert on.arguments_json is not None and '"Paris"' in on.arguments_json assert on.result_json is not None and "21" in on.result_json @@ -689,9 +686,7 @@ def test_content_capture_gated_off_by_default(): payload = _sample_payload( messages=[{"role": "user", "content": "secret prompt"}], ) - payload["response"]["choices"] = [ - {"finish_reason": "stop", "message": {"role": "assistant", "content": "secret"}} - ] + payload["response"]["choices"] = [{"finish_reason": "stop", "message": {"role": "assistant", "content": "secret"}}] data = LLMCallSpanData.from_standard_logging_payload(payload) assert data.messages_in == () assert data.choices_out == () @@ -911,6 +906,330 @@ def test_ocr_pages_without_markdown_stay_empty(): assert data.choices_out == () +def _assistant_choice(content: str, finish_reason: str | None = None) -> dict[str, object]: + return { + "message": {"role": "assistant", "content": content, "refusal": None, "tool_calls": None}, + "finish_reason": finish_reason, + } + + +def _route_payload(call_type: str, model: str, response: Mapping[str, object]) -> dict[str, object]: + return _sample_payload(call_type=call_type, model=model, messages=None, response=response) + + +def test_text_completion_choices_become_assistant_messages_in_choice_order() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "atext_completion", + "gpt-3.5-turbo-instruct", + { + "id": "cmpl-1", + "object": "text_completion", + "choices": [ + {"index": 0, "text": " first", "finish_reason": "length", "logprobs": None}, + {"index": 1, "text": " second", "finish_reason": "stop", "logprobs": None}, + ], + }, + ), + capture_content=True, + ) + + assert data.choices_out == ( + {"index": 0, "logprobs": None, **_assistant_choice(" first", "length")}, + {"index": 1, "logprobs": None, **_assistant_choice(" second", "stop")}, + ) + assert data.finish_reasons == ("length", "stop") + assert data.response_id == "cmpl-1" + + +def test_text_completion_choices_keep_provider_fields_beside_the_synthesized_message() -> None: + choice: Final = { + "index": 2, + "text": "Hello there", + "finish_reason": "stop", + "logprobs": {"tokens": ["Hello"], "token_logprobs": [-0.1]}, + "content_filter_results": {"hate": {"filtered": False}}, + "provider_specific": {"cached": True}, + } + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("atext_completion", "gpt-3.5-turbo-instruct", {"choices": [choice]}), capture_content=True + ) + + assert data.choices_out == ( + {k: v for k, v in choice.items() if k != "text"} | _assistant_choice("Hello there", "stop"), + ) + + +def test_text_completion_choices_follow_the_content_capture_gate_but_finish_reasons_do_not() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "atext_completion", "gpt-3.5-turbo-instruct", {"choices": [{"text": "x", "finish_reason": "stop"}]} + ) + ) + + assert data.choices_out == () + assert data.finish_reasons == ("stop",) + + +def test_chat_choices_with_a_message_are_passed_through_untouched_even_beside_a_stray_text_key() -> None: + choice: Final = { + "index": 0, + "finish_reason": "stop", + "text": "no", + "message": {"role": "assistant", "content": "chat"}, + } + data: Final = LLMCallSpanData.from_standard_logging_payload( + _sample_payload(response={"choices": [choice]}), capture_content=True + ) + + assert data.choices_out == (choice,) + + +def test_transcription_text_becomes_one_assistant_choice() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("atranscription", "gpt-4o-mini-transcribe", {"text": "What is the weather like?", "task": "x"}), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice("What is the weather like?"),) + assert data.finish_reasons == () + + +def test_empty_transcription_text_stays_empty() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("atranscription", "gpt-4o-mini-transcribe", {"text": ""}), capture_content=True + ) + + assert data.choices_out == () + + +def test_moderation_results_become_one_verdict_per_input_naming_the_hit_categories() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "amoderation", + "omni-moderation-latest", + { + "id": "modr-1", + "results": [ + { + "flagged": True, + "categories": {"harassment": False, "violence": True, "self-harm": True}, + "category_scores": {"harassment": 0.01, "violence": 0.98, "self-harm": 0.7}, + }, + {"flagged": False, "categories": {"violence": False}}, + {"flagged": True}, + ], + }, + ), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice("flagged: violence, self-harm\n\nnot flagged\n\nflagged"),) + assert data.response_id == "modr-1" + + +def test_moderation_output_follows_the_content_capture_gate() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("amoderation", "omni-moderation-latest", {"results": [{"flagged": True}]}) + ) + + assert data.choices_out == () + + +def test_moderation_results_without_a_verdict_produce_no_output() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("amoderation", "omni-moderation-latest", {"results": [{"categories": {"violence": True}}]}), + capture_content=True, + ) + + assert data.choices_out == () + + +def test_rerank_results_become_ranked_indices_and_scores_with_the_document_text() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "arerank", + "rerank-v4.0-fast", + { + "id": "rr-1", + "results": [ + {"index": 2, "relevance_score": 0.91, "document": {"text": "Paris is the capital of France."}}, + {"index": 0, "relevance_score": 0.07}, + {"index": 1, "relevance_score": 0.02, "document": "not-a-document"}, + ], + "meta": {"billed_units": {"search_units": 1}}, + }, + ), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice("[2] 0.91\nParis is the capital of France.\n\n[0] 0.07\n\n[1] 0.02"),) + assert data.finish_reasons == () + assert data.response_id == "rr-1" + + +def test_rerank_output_follows_the_content_capture_gate() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("arerank", "rerank-v4.0-fast", {"results": [{"index": 0, "relevance_score": 0.5}]}) + ) + + assert data.choices_out == () + + +def test_rerank_results_without_an_index_and_score_produce_no_output() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "arerank", + "rerank-v4.0-fast", + {"results": [{"index": 0, "document": {"text": "x"}}, {"relevance_score": 0.5}, "not-a-result"]}, + ), + capture_content=True, + ) + + assert data.choices_out == () + + +def test_search_results_become_title_url_and_snippet_blocks_in_result_order() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "asearch", + "exa-search", + { + "object": "search", + "results": [ + {"title": "Eiffel Tower", "url": "https://example.com/eiffel", "snippet": "A lattice tower."}, + {"url": "https://example.com/bare", "date": "2024-01-01"}, + {"title": "no url", "snippet": "kept"}, + ], + }, + ), + capture_content=True, + ) + + assert data.choices_out == ( + _assistant_choice( + "Eiffel Tower\nhttps://example.com/eiffel\nA lattice tower.\n\nhttps://example.com/bare\n\nno url\nkept" + ), + ) + assert data.finish_reasons == () + + +def test_search_output_follows_the_content_capture_gate() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("asearch", "exa-search", {"results": [{"url": "https://example.com"}]}) + ) + + assert data.choices_out == () + + +def test_search_results_without_any_text_field_produce_no_output() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "asearch", "exa-search", {"results": [{"date": "2024-01-01"}, {"title": "", "url": None}, "not-a-result"]} + ), + capture_content=True, + ) + + assert data.choices_out == () + + +def test_image_data_becomes_a_size_summary_and_never_carries_the_base64_payload() -> None: + encoded: Final = "QUJDRA==" + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "aimage_generation", + "gpt-image-1-mini", + { + "created": 1, + "data": [ + {"b64_json": encoded, "revised_prompt": "a red bicycle"}, + {"url": "https://images.example/cat.png"}, + {"b64_json": "QUJDREVGR0g="}, + ], + }, + ), + capture_content=True, + ) + + assert data.choices_out == ( + _assistant_choice( + "a red bicycle\nb64_json image (4 bytes)\n\nhttps://images.example/cat.png\n\nb64_json image (8 bytes)" + ), + ) + assert encoded not in json.dumps(data.choices_out) + + +def test_image_data_without_a_url_or_payload_stays_empty_and_embeddings_are_not_images() -> None: + images: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("aimage_generation", "gpt-image-1-mini", {"data": [{"revised_prompt": "x"}]}), + capture_content=True, + ) + embeddings: Final = LLMCallSpanData.from_standard_logging_payload( + _embedding_payload([[0.1, 0.2]]), capture_content=True + ) + + assert images.choices_out == () + assert embeddings.choices_out == () + + +def test_speech_summary_becomes_a_media_type_and_byte_count_choice() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload( + "aspeech", "gpt-4o-mini-tts", {"object": "binary", "content_type": "audio/mpeg", "num_bytes": 48210} + ), + capture_content=True, + ) + + assert data.choices_out == (_assistant_choice("audio/mpeg (48210 bytes)"),) + + +def test_speech_summary_without_a_media_type_is_the_byte_count_and_follows_the_capture_gate() -> None: + response: Final = {"object": "binary", "content_type": None, "num_bytes": 7} + shown: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("aspeech", "gpt-4o-mini-tts", response), capture_content=True + ) + gated: Final = LLMCallSpanData.from_standard_logging_payload(_route_payload("aspeech", "gpt-4o-mini-tts", response)) + + assert shown.choices_out == (_assistant_choice("7 bytes"),) + assert gated.choices_out == () + + +def test_speech_response_without_a_byte_count_produces_no_output() -> None: + data: Final = LLMCallSpanData.from_standard_logging_payload( + _route_payload("aspeech", "gpt-4o-mini-tts", {"object": "binary", "content_type": "audio/mpeg"}), + capture_content=True, + ) + + assert data.choices_out == () + + +def test_speech_binary_response_is_logged_as_its_summary_not_dropped() -> None: + import httpx + + from litellm.litellm_core_utils.litellm_logging import _extract_response_obj_and_hidden_params + from litellm.types.llms.openai import HttpxBinaryResponseContent + + raw: Final = httpx.Response(200, headers={"content-type": "audio/mpeg"}, content=b"\x00" * 1234) + response_obj, hidden_params = _extract_response_obj_and_hidden_params(HttpxBinaryResponseContent(raw), None) + + assert response_obj == {"object": "binary", "content_type": "audio/mpeg", "num_bytes": 1234} + assert hidden_params is None + + +def test_speech_binary_response_still_streaming_reports_the_bytes_downloaded_so_far() -> None: + import httpx + + from litellm.types.llms.openai import HttpxBinaryResponseContent + + unread: Final = httpx.Response(200, stream=httpx.ByteStream(b"\x00" * 10)) + + assert HttpxBinaryResponseContent(unread).logging_summary() == { + "object": "binary", + "content_type": None, + "num_bytes": 0, + } + + def test_request_identity_prefers_canonical_team_keys(): from litellm.integrations.otel.model.payloads import RequestIdentity @@ -931,9 +1250,7 @@ def test_request_identity_prefers_canonical_team_keys(): def test_request_identity_falls_back_to_legacy_team_keys(): from litellm.integrations.otel.model.payloads import RequestIdentity - payload = _sample_payload( - metadata={"team_id": "legacy-team", "team_alias": "legacy"} - ) + payload = _sample_payload(metadata={"team_id": "legacy-team", "team_alias": "legacy"}) ident = RequestIdentity.from_payload(payload) assert ident.team_id == "legacy-team" assert ident.team_alias == "legacy" @@ -952,7 +1269,10 @@ def test_request_identity_falls_back_to_legacy_team_keys(): }, "from-header", ), - ({"proxy_server_request": {"headers": {"langfuse_trace_name": ""}}, "metadata": {"trace_name": "body"}}, "body"), + ( + {"proxy_server_request": {"headers": {"langfuse_trace_name": ""}}, "metadata": {"trace_name": "body"}}, + "body", + ), ({"proxy_server_request": {"headers": {}}, "metadata": {"user_api_key_team_id": "t1"}}, None), ({}, None), ], @@ -1002,7 +1322,15 @@ def test_caller_trace_name_prefers_the_langfuse_header_over_body_metadata(reques ), ({}, TraceControls()), ], - ids=["body", "headers-beat-body", "anthropic-body", "non-string-tags-dropped", "scalar-coercion", "mutation-controls-ignored", "empty"], + ids=[ + "body", + "headers-beat-body", + "anthropic-body", + "non-string-tags-dropped", + "scalar-coercion", + "mutation-controls-ignored", + "empty", + ], ) def test_caller_trace_controls_carry_user_session_and_tags(request_data, expected): assert caller_trace_controls({"litellm_params": request_data}) == expected @@ -1168,9 +1496,7 @@ def test_content_capture_opt_in_retains_bodies(): payload = _sample_payload( messages=[{"role": "user", "content": "secret prompt"}], ) - payload["response"]["choices"] = [ - {"finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}} - ] + payload["response"]["choices"] = [{"finish_reason": "stop", "message": {"role": "assistant", "content": "hi"}}] data = LLMCallSpanData.from_standard_logging_payload(payload, capture_content=True) assert data.messages_in and data.messages_in[0]["content"] == "secret prompt" assert data.choices_out and data.choices_out[0]["message"]["content"] == "hi" @@ -1187,41 +1513,17 @@ def test_capture_span_content_resolves_modes(): # default (no_content) → off assert OpenTelemetryV2Config().capture_span_content is False + assert OpenTelemetryV2Config(capture_message_content=CaptureMessageContent.SPAN_ONLY).capture_span_content is True assert ( - OpenTelemetryV2Config( - capture_message_content=CaptureMessageContent.SPAN_ONLY - ).capture_span_content - is True - ) - assert ( - OpenTelemetryV2Config( - capture_message_content=CaptureMessageContent.SPAN_AND_EVENT - ).capture_span_content - is True + OpenTelemetryV2Config(capture_message_content=CaptureMessageContent.SPAN_AND_EVENT).capture_span_content is True ) # event-only does not authorize span-attribute content - assert ( - OpenTelemetryV2Config( - capture_message_content=CaptureMessageContent.EVENT_ONLY - ).capture_span_content - is False - ) + assert OpenTelemetryV2Config(capture_message_content=CaptureMessageContent.EVENT_ONLY).capture_span_content is False # V1 accepted UPPER_SNAKE_CASE; the env value is case-insensitive so an # operator carrying ``SPAN_AND_EVENT`` forward still enables capture. - assert ( - OpenTelemetryV2Config( - capture_message_content="SPAN_AND_EVENT" - ).capture_span_content - is True - ) - assert ( - OpenTelemetryV2Config(capture_message_content="SPAN_ONLY").capture_span_content - is True - ) - assert ( - OpenTelemetryV2Config(capture_message_content="NO_CONTENT").capture_span_content - is False - ) + assert OpenTelemetryV2Config(capture_message_content="SPAN_AND_EVENT").capture_span_content is True + assert OpenTelemetryV2Config(capture_message_content="SPAN_ONLY").capture_span_content is True + assert OpenTelemetryV2Config(capture_message_content="NO_CONTENT").capture_span_content is False def test_capture_message_content_normalizer_only_touches_strings(): diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py b/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py index 9fa198c4ec5..1e2ae24a329 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_vendor_mappers.py @@ -6,6 +6,8 @@ backends, so one trace lights up every configured destination. """ import json +from collections.abc import Mapping +from typing import Final import pytest @@ -249,6 +251,34 @@ def test_langfuse_mapper_renders_an_ocr_call_with_the_page_markdown_as_output(): assert attrs["langfuse.observation.type"] == "generation" +@pytest.mark.parametrize( + ("call_type", "response", "expected_content"), + [ + ("atext_completion", {"choices": [{"text": "Paris.", "finish_reason": "stop"}]}, "Paris."), + ("atranscription", {"text": "What is the weather like?"}, "What is the weather like?"), + ("amoderation", {"results": [{"flagged": True, "categories": {"violence": True}}]}, "flagged: violence"), + ("aimage_generation", {"data": [{"b64_json": "QUJDRA=="}]}, "b64_json image (4 bytes)"), + ("aspeech", {"object": "binary", "content_type": "audio/mpeg", "num_bytes": 9}, "audio/mpeg (9 bytes)"), + ], +) +def test_langfuse_mapper_renders_every_non_chat_route_output_as_an_assistant_message( + call_type: str, response: Mapping[str, object], expected_content: str +) -> None: + payload: Final[dict[str, object]] = { + "call_type": call_type, + "custom_llm_provider": "openai", + "model": "m", + "messages": None, + "response": response, + } + attrs: Final = LangfuseMapper().map(LLMCallSpanData.from_standard_logging_payload(payload, capture_content=True)) + + assert json.loads(attrs["langfuse.observation.output"]) == [ + {"role": "assistant", "content": expected_content, "refusal": None, "tool_calls": None} + ] + assert attrs["langfuse.observation.type"] == "generation" + + # --------------------------------------------------------------------------- # # Weave # --------------------------------------------------------------------------- # diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py index 7bf4533979a..f787d370f04 100644 --- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py +++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py @@ -1664,7 +1664,7 @@ class TestEnableAnthropicPromptCaching: points = self._points(model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", provider="bedrock") assert [p["index"] for p in points] == [None, -1] - @pytest.mark.parametrize("model, provider", [("gpt-4o", "openai"), ("gemini-2.0-flash", "gemini")]) + @pytest.mark.parametrize("model, provider", [("gpt-4o", "openai")]) def test_non_anthropic_providers_never_injected(self, monkeypatch, model, provider): """These report supports_prompt_caching=True but never consume cache_control markers.""" from litellm.utils import supports_prompt_caching diff --git a/tests/test_litellm/integrations/test_langsmith_init.py b/tests/test_litellm/integrations/test_langsmith_init.py index f56d2310e73..9c0650baee6 100644 --- a/tests/test_litellm/integrations/test_langsmith_init.py +++ b/tests/test_litellm/integrations/test_langsmith_init.py @@ -1,13 +1,17 @@ import asyncio +import json import os +from datetime import datetime, timezone +from decimal import Decimal from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest - import litellm from litellm.integrations.langsmith import LangsmithLogger +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.integrations.langsmith import LangsmithQueueObject @@ -219,6 +223,121 @@ class TestLangsmithLoggerInit: assert len(logger.log_queue) == 1 +class TestLangsmithBatchSerialization: + async def _logger(self, transport_handler, tenant_id=None): + logger = LangsmithLogger( + langsmith_api_key="test-key", + langsmith_project="test-project", + langsmith_base_url="https://api.smith.langchain.com", + langsmith_tenant_id=tenant_id, + ) + if logger._flush_task is not None: + logger._flush_task.cancel() + handler = AsyncHTTPHandler() + await handler.client.aclose() + handler.client = httpx.AsyncClient( + transport=httpx.MockTransport(transport_handler) + ) + logger.async_httpx_client = handler + return logger + + @staticmethod + def _capturing_transport(captured): + async def handle(request: httpx.Request) -> httpx.Response: + captured.append(request) + return httpx.Response(200, request=request, json={"ok": True}) + + return handle + + def _queue(self, logger, extra): + return [ + LangsmithQueueObject( + data={"id": "run-1", "name": "LLMRun", "extra": extra}, + credentials=logger.default_credentials, + ) + ] + + @pytest.mark.asyncio + async def test_datetime_and_decimal_metadata_reach_langsmith_as_strings(self): + captured: list[httpx.Request] = [] # mutable-ok: transport capture buffer + logger = await self._logger(self._capturing_transport(captured)) + logger.log_queue = self._queue( + logger, + { + "created_at": datetime(2026, 1, 2, 3, 4, 5, tzinfo=timezone.utc), + "spend": Decimal("0.0042"), + }, + ) + + await logger.async_send_batch() + + assert len(captured) == 1, "batch was dropped instead of being sent" + body = json.loads(captured[0].content) + assert body["post"][0]["extra"] == { + "created_at": "2026-01-02 03:04:05+00:00", + "spend": "0.0042", + } + await logger.async_httpx_client.client.aclose() + + @pytest.mark.asyncio + async def test_nan_metadata_is_dropped_instead_of_shipping_invalid_json(self): + captured: list[httpx.Request] = [] # mutable-ok: transport capture buffer + logger = await self._logger(self._capturing_transport(captured)) + logger.log_queue = self._queue(logger, {"score": float("nan")}) + + await logger.async_send_batch() + + assert captured == [], ( + "nan metadata must abort the batch: a bare NaN token is invalid JSON and LangSmith rejects it" + ) + await logger.async_httpx_client.client.aclose() + + @pytest.mark.asyncio + async def test_batch_declares_json_content_type(self): + captured: list[httpx.Request] = [] # mutable-ok: transport capture buffer + logger = await self._logger(self._capturing_transport(captured)) + logger.log_queue = self._queue(logger, {"model": "gpt-4.1-mini"}) + + await logger.async_send_batch() + + assert captured[0].headers["content-type"] == "application/json", ( + "a content= body carries no implicit content type; LangSmith refuses it without this header" + ) + assert captured[0].url.path.endswith("/api/v1/runs/batch") + assert captured[0].headers["x-api-key"] == "test-key" + assert "x-tenant-id" not in captured[0].headers + await logger.async_httpx_client.client.aclose() + + @pytest.mark.asyncio + async def test_tenant_id_is_forwarded_on_the_batch_request(self): + captured: list[httpx.Request] = [] # mutable-ok: transport capture buffer + logger = await self._logger( + self._capturing_transport(captured), tenant_id="tenant-1" + ) + logger.log_queue = self._queue(logger, {"model": "gpt-4.1-mini"}) + + await logger.async_send_batch() + + assert captured[0].headers["x-tenant-id"] == "tenant-1" + await logger.async_httpx_client.client.aclose() + + @pytest.mark.asyncio + async def test_langsmith_error_response_does_not_propagate(self): + captured: list[httpx.Request] = [] # mutable-ok: transport capture buffer + + async def reject(request: httpx.Request) -> httpx.Response: + captured.append(request) + return httpx.Response(422, request=request, text="bad run") + + logger = await self._logger(reject) + logger.log_queue = self._queue(logger, {"model": "gpt-4.1-mini"}) + + await logger.async_send_batch() + + assert len(captured) == 1, "the batch never left the process" + await logger.async_httpx_client.client.aclose() + + class TestLangsmithPrepareLogData: """Regression test for #24001: _prepare_log_data must inject usage_metadata into outputs so LangSmith's Cost column is populated.""" @@ -544,12 +663,10 @@ async def test_events_appended_during_flush_are_not_dropped(): credentials=logger.default_credentials, data={"id": "late"} ) - async def fake_post( - url: str, json: dict[str, list[dict[str, str]]], headers: dict[str, str] - ) -> MagicMock: + async def fake_post(url: str, content: str, headers: dict[str, str]) -> MagicMock: if not sent_batches: logger.log_queue.append(late_event) - sent_batches.append(json["post"]) + sent_batches.append(json.loads(content)["post"]) response = MagicMock() response.status_code = 200 response.raise_for_status = MagicMock() diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py index bea9a38e9dd..974961f2eb5 100644 --- a/tests/test_litellm/integrations/test_opentelemetry.py +++ b/tests/test_litellm/integrations/test_opentelemetry.py @@ -1,5 +1,6 @@ import asyncio import concurrent.futures +import contextlib import gc import json import os @@ -9,21 +10,30 @@ import time import unittest import weakref from datetime import datetime, timedelta, timezone +from pathlib import Path from types import MappingProxyType -from parameterized import parameterized +from typing import Final from unittest.mock import MagicMock, patch +import pytest + # Adds the grandparent directory to sys.path to allow importing project modules from opentelemetry import trace -from opentelemetry.sdk._logs import LogData +from opentelemetry._logs.severity import SeverityNumber +from opentelemetry.sdk._logs import LogData, LogRecord from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider -from opentelemetry.sdk._logs.export import InMemoryLogExporter, SimpleLogRecordProcessor +from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogExportResult, SimpleLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader, MetricsData -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult +from opentelemetry.sdk.util.instrumentation import InstrumentationScope from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from parameterized import parameterized +import requests + +from conftest import TlsSink, write_self_signed_cert import litellm from litellm.integrations import opentelemetry as otel_module from litellm.integrations.opentelemetry import ( @@ -2081,6 +2091,138 @@ class TestOpenTelemetryEndpointNormalization(unittest.TestCase): self.assertEqual(traces, "http://collector:4318/v1/traces") +def _isolate_otlp_tls_env(monkeypatch: pytest.MonkeyPatch) -> None: + for key in ( + "SSL_VERIFY", + "SSL_CERT_FILE", + "OTEL_EXPORTER_OTLP_CERTIFICATE", + "OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", + "OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE", + "OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE", + ): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TIMEOUT", "2") + monkeypatch.setattr(litellm, "ssl_verify", True) + + +def _ended_span() -> tuple[TracerProvider, ReadableSpan]: + provider: Final = TracerProvider() + span = provider.get_tracer(__name__).start_span("tls-export-test") + span.end() + return provider, span + + +def _assert_export_rejected(processor, span: ReadableSpan, sink: TlsSink) -> None: + with contextlib.suppress(requests.exceptions.SSLError): + result: Final = processor.span_exporter.export([span]) + assert result is SpanExportResult.FAILURE, f"rejected export must report failure, got {result}" + assert sink.received.empty(), "sink received a request it should never have trusted" + + +def _otlp_http_otel(endpoint: str) -> OpenTelemetry: + return OpenTelemetry(config=OpenTelemetryConfig(exporter="otlp_http", endpoint=endpoint)) + + +def test_otlp_http_span_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None: + _isolate_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path) + otel: Final = _otlp_http_otel(tls_sink.url) + processor: Final = otel._get_span_processor() + provider, span = _ended_span() + try: + result: Final = processor.span_exporter.export([span]) + assert result is SpanExportResult.SUCCESS, f"span export failed: {result}" + assert tls_sink.received.get(timeout=5) == "/v1/traces" + finally: + processor.shutdown() + provider.shutdown() + + +def test_otlp_http_metric_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None: + _isolate_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path) + otel: Final = _otlp_http_otel(tls_sink.url) + reader: Final = otel._get_metric_reader() + provider: Final = MeterProvider(metric_readers=[reader]) + try: + provider.get_meter(__name__).create_counter("tls_export_test").add(1) + assert provider.force_flush(), "metric flush failed" + assert tls_sink.received.get(timeout=5) == "/v1/metrics" + finally: + provider.shutdown() + + +def test_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink) -> None: + _isolate_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path) + otel: Final = _otlp_http_otel(tls_sink.url) + exporter: Final = otel._get_log_exporter() + try: + record: Final = LogRecord( + timestamp=int(time.time() * 1e9), + observed_timestamp=int(time.time() * 1e9), + trace_id=0, + span_id=0, + trace_flags=trace.TraceFlags(0), + severity_number=SeverityNumber.INFO, + body="tls-export-test", + ) + log_data: Final = LogData(log_record=record, instrumentation_scope=InstrumentationScope("tls-export-test")) + result: Final = exporter.export([log_data]) + assert result is LogExportResult.SUCCESS, f"log export failed: {result}" + assert tls_sink.received.get(timeout=5) == "/v1/logs" + finally: + exporter.shutdown() + + +def test_otlp_http_export_skips_verification_when_ssl_verify_false( + monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink +) -> None: + _isolate_otlp_tls_env(monkeypatch) + monkeypatch.setenv("SSL_VERIFY", "false") + otel: Final = _otlp_http_otel(tls_sink.url) + processor: Final = otel._get_span_processor() + provider, span = _ended_span() + try: + result: Final = processor.span_exporter.export([span]) + assert result is SpanExportResult.SUCCESS, f"span export failed: {result}" + assert tls_sink.received.get(timeout=5) == "/v1/traces" + finally: + processor.shutdown() + provider.shutdown() + + +def test_otlp_http_export_rejects_untrusted_collector_by_default( + monkeypatch: pytest.MonkeyPatch, tls_sink: TlsSink +) -> None: + _isolate_otlp_tls_env(monkeypatch) + otel: Final = _otlp_http_otel(tls_sink.url) + processor: Final = otel._get_span_processor() + provider, span = _ended_span() + try: + _assert_export_rejected(processor, span, tls_sink) + finally: + processor.shutdown() + provider.shutdown() + + +def test_otel_certificate_env_takes_precedence_over_ssl_cert_file( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, tls_sink: TlsSink +) -> None: + _isolate_otlp_tls_env(monkeypatch) + unrelated_certificate, _ = write_self_signed_cert(tmp_path, "unrelated") + monkeypatch.setenv("SSL_CERT_FILE", tls_sink.certificate_path) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_CERTIFICATE", str(unrelated_certificate)) + otel: Final = _otlp_http_otel(tls_sink.url) + processor: Final = otel._get_span_processor() + provider, span = _ended_span() + try: + _assert_export_rejected(processor, span, tls_sink) + finally: + processor.shutdown() + provider.shutdown() + + class TestOpenTelemetryProtocolSelection(unittest.TestCase): """Test suite for verifying correct exporter selection based on protocol""" diff --git a/tests/test_litellm/integrations/test_prometheus_deployment_state_proxy_rejects.py b/tests/test_litellm/integrations/test_prometheus_deployment_state_proxy_rejects.py new file mode 100644 index 00000000000..fedfc2b0848 --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_deployment_state_proxy_rejects.py @@ -0,0 +1,101 @@ +""" +LIT-7701 attributes a pre_call_hook rejection's failure log to the model +group's single deployment (``model_id`` and ``custom_llm_provider``) and flags +it with ``PROXY_REJECTED_BEFORE_ROUTING_KEY``. The deployment health metrics +must keep treating such rejects as "no deployment picked": a key rate limit or +guardrail block never reached the deployment, so it must not flip +``litellm_deployment_state`` to partial outage or count as a deployment failure +response. A failure raised after the router picked a deployment (a post-call +guardrail block, a provider error) carries no flag and keeps its deployment labels. +""" + +import pytest +from fastapi import HTTPException +from prometheus_client import REGISTRY + +from litellm.constants import PROXY_REJECTED_BEFORE_ROUTING_KEY +from litellm.exceptions import GuardrailRaisedException +from litellm.integrations.prometheus import PrometheusLogger +from litellm.proxy._types import ProxyException +from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + +@pytest.fixture(autouse=True) +def cleanup_prometheus_registry(): + for collector in list(REGISTRY._collector_to_names.keys()): + try: + REGISTRY.unregister(collector) + except Exception: + pass + + yield + + for collector in list(REGISTRY._collector_to_names.keys()): + try: + REGISTRY.unregister(collector) + except Exception: + pass + + +def _attributed_failure_kwargs(exception: Exception, rejected_before_routing: bool) -> dict: + return { + "model": "openai/gpt-4.1", + "litellm_params": { + "custom_llm_provider": "openai", + "metadata": {"model_info": {"id": "dep-1"}, "model_group": "internal-model"}, + **({PROXY_REJECTED_BEFORE_ROUTING_KEY: True} if rejected_before_routing else {}), + }, + "standard_logging_object": { + "model_id": "dep-1", + "model_group": "internal-model", + "api_base": "https://api.openai.com", + "metadata": {}, + }, + "exception": exception, + } + + +def _model_id_values(metric) -> set[str]: + index = metric._labelnames.index("model_id") + return {sample_key[index] for sample_key in metric._metrics} + + +class _ProviderError(Exception): + status_code = 500 + + +@pytest.mark.parametrize( + "rejection", + [ + HTTPException(status_code=403, detail="guardrail blocked"), + ProxyException(message="budget exceeded", type="budget_exceeded", param=None, code=400), + ProxyRateLimitError(detail={"error": "key rpm limit"}), + GuardrailRaisedException(guardrail_name="pii", message="blocked", status_code=403), + ], + ids=["http_exception", "proxy_exception", "proxy_rate_limit", "guardrail_raised"], +) +def test_attributed_proxy_reject_leaves_deployment_healthy(rejection: Exception): + logger = PrometheusLogger() + + logger.set_llm_deployment_failure_metrics(_attributed_failure_kwargs(rejection, rejected_before_routing=True)) + + assert logger.litellm_deployment_state._metrics == {} + assert _model_id_values(logger.litellm_deployment_failure_responses) == {""} + assert _model_id_values(logger.litellm_deployment_total_requests) == {""} + + +@pytest.mark.parametrize( + "failure", + [ + _ProviderError("upstream 500"), + GuardrailRaisedException(guardrail_name="pii", message="response blocked", status_code=400), + ], + ids=["provider_error", "post_call_guardrail"], +) +def test_failure_after_routing_still_marks_deployment_partial_outage(failure: Exception): + logger = PrometheusLogger() + + logger.set_llm_deployment_failure_metrics(_attributed_failure_kwargs(failure, rejected_before_routing=False)) + + assert _model_id_values(logger.litellm_deployment_state) == {"dep-1"} + assert _model_id_values(logger.litellm_deployment_failure_responses) == {"dep-1"} diff --git a/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py b/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py index 519a13751f1..f86f4460b28 100644 --- a/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py +++ b/tests/test_litellm/integrations/test_prometheus_requested_model_cardinality.py @@ -116,6 +116,22 @@ async def test_unknown_models_collapse_to_one_series_on_proxy_request_metrics(ro assert _total_value(metric) == 25 +@pytest.mark.asyncio +@pytest.mark.parametrize("model", [["gpt-4o-mini"], {"name": "gpt-4o-mini"}, 123]) +async def test_non_string_models_collapse_to_other_on_proxy_request_metrics(router, model: object): + logger = PrometheusLogger() + + with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await logger.async_post_call_failure_hook( + request_data={"model": model, "metadata": {}, "proxy_server_request": {}}, + original_exception=_ClientSideError("'model' must be a string."), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key-1"), + ) + + assert _requested_model_values(logger.litellm_proxy_failed_requests_metric) == {UNRECOGNIZED_REQUESTED_MODEL_LABEL} + assert _total_value(logger.litellm_proxy_failed_requests_metric) == 1 + + @pytest.mark.asyncio async def test_known_alias_and_wildcard_models_keep_their_own_labels(router): logger = PrometheusLogger() diff --git a/tests/test_litellm/integrations/test_prometheus_zero_cost_metric.py b/tests/test_litellm/integrations/test_prometheus_zero_cost_metric.py new file mode 100644 index 00000000000..989009b309a --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_zero_cost_metric.py @@ -0,0 +1,179 @@ +import datetime +from typing import Final + +import pytest +from prometheus_client import REGISTRY +from prometheus_client.samples import Sample + +import litellm +from litellm.integrations.prometheus import PrometheusLogger +from litellm.types.utils import StandardLoggingZeroCostDiagnostic + +METRIC: Final = "litellm_zero_cost_requests_total" +MISSING_KEY_DIAGNOSTIC: Final[StandardLoggingZeroCostDiagnostic] = { + "reason": "missing_pricing_key", + "pricing_model": "dep-1", + "missing_pricing_keys": ("input_cost_per_token", "output_cost_per_token"), +} + + +def _clear_prometheus_registry() -> None: + for collector in list(REGISTRY._collector_to_names.keys()): + try: + REGISTRY.unregister(collector) + except Exception: + pass + + +def _samples(metric_name: str) -> list[Sample]: + return [sample for metric in REGISTRY.collect() for sample in metric.samples if sample.name == metric_name] + + +def _payload(zero_cost_diagnostic: StandardLoggingZeroCostDiagnostic | None) -> dict[str, object]: + return { + "id": "t", + "call_type": "completion", + "response_cost": 0.0, + "status": "success", + "total_tokens": 30, + "prompt_tokens": 20, + "completion_tokens": 10, + "startTime": 1.0, + "endTime": 2.0, + "completionStartTime": 1.5, + "model": "openai/gpt-5.4-nano", + "model_id": "dep-1", + "model_group": "per-second-priced-chat", + "api_base": "https://api.openai.com", + "custom_llm_provider": "openai", + "request_tags": [], + "end_user": None, + "cache_hit": False, + "stream": False, + "response": {"id": "chatcmpl-1"}, + "model_parameters": {}, + "zero_cost_diagnostic": zero_cost_diagnostic, + "metadata": { + "user_api_key_hash": "h", + "user_api_key_alias": "a", + "user_api_key_team_id": "t", + "user_api_key_team_alias": "ta", + "user_api_key_user_id": "u", + "user_api_key_user_email": "e@x.com", + "user_api_key_org_id": None, + "user_api_key_org_alias": None, + "requester_metadata": None, + "user_api_key_end_user_id": None, + "usage_object": None, + }, + "hidden_params": {"litellm_overhead_time_ms": None, "additional_headers": None}, + } + + +async def _log_success( + logger: PrometheusLogger, zero_cost_diagnostic: StandardLoggingZeroCostDiagnostic | None +) -> None: + now: Final = datetime.datetime.now() + kwargs: Final = { + "model": "openai/gpt-5.4-nano", + "litellm_params": {"metadata": {}}, + "standard_logging_object": _payload(zero_cost_diagnostic), + "stream": False, + "start_time": now - datetime.timedelta(seconds=3), + "api_call_start_time": now - datetime.timedelta(seconds=2), + "completion_start_time": now - datetime.timedelta(seconds=1), + "end_time": now, + } + await logger.async_log_success_event(kwargs, None, now, now) + + +async def _log_failure( + logger: PrometheusLogger, zero_cost_diagnostic: StandardLoggingZeroCostDiagnostic | None +) -> None: + now: Final = datetime.datetime.now() + kwargs: Final = { + "model": "openai/gpt-5.4-nano", + "litellm_params": {"metadata": {}}, + "standard_logging_object": {**_payload(zero_cost_diagnostic), "status": "failure"}, + "exception": Exception("stream cut off after the usage chunk"), + "stream": True, + "start_time": now - datetime.timedelta(seconds=3), + "end_time": now, + } + await logger.async_log_failure_event(kwargs, None, now, now) + + +@pytest.mark.asyncio +async def test_failure_event_counts_a_zero_cost_request_by_model_and_reason() -> None: + _clear_prometheus_registry() + try: + logger: Final = PrometheusLogger() + await _log_failure(logger, None) + assert _samples(METRIC) == [] + + await _log_failure(logger, MISSING_KEY_DIAGNOSTIC) + + samples: Final = _samples(METRIC) + assert len(samples) == 1 + assert samples[0].labels == { + "requested_model": "per-second-priced-chat", + "model": "openai/gpt-5.4-nano", + "model_id": "dep-1", + "api_provider": "openai", + "reason": "missing_pricing_key", + } + assert samples[0].value == 1.0 + finally: + _clear_prometheus_registry() + + +@pytest.mark.asyncio +async def test_success_event_counts_a_zero_cost_request_by_model_and_reason() -> None: + _clear_prometheus_registry() + try: + logger: Final = PrometheusLogger() + await _log_success(logger, MISSING_KEY_DIAGNOSTIC) + await _log_success(logger, MISSING_KEY_DIAGNOSTIC) + + samples: Final = _samples(METRIC) + assert len(samples) == 1 + assert samples[0].labels == { + "requested_model": "per-second-priced-chat", + "model": "openai/gpt-5.4-nano", + "model_id": "dep-1", + "api_provider": "openai", + "reason": "missing_pricing_key", + } + assert samples[0].value == 2.0 + finally: + _clear_prometheus_registry() + + +@pytest.mark.asyncio +async def test_request_without_a_diagnostic_leaves_the_counter_untouched() -> None: + _clear_prometheus_registry() + try: + await _log_success(PrometheusLogger(), None) + + assert _samples(METRIC) == [] + finally: + _clear_prometheus_registry() + + +@pytest.mark.asyncio +async def test_label_filter_that_drops_reason_still_counts_the_request() -> None: + _clear_prometheus_registry() + previous_config: Final = litellm.prometheus_metrics_config + litellm.prometheus_metrics_config = [ + {"group": "zero_cost", "metrics": [METRIC], "include_labels": ["requested_model"]} + ] + try: + await _log_success(PrometheusLogger(), MISSING_KEY_DIAGNOSTIC) + + samples: Final = _samples(METRIC) + assert len(samples) == 1 + assert samples[0].labels == {"requested_model": "per-second-priced-chat"} + assert samples[0].value == 1.0 + finally: + litellm.prometheus_metrics_config = previous_config + _clear_prometheus_registry() diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py index f39f41a6d12..d6450d0f1de 100644 --- a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py @@ -287,6 +287,162 @@ async def test_execute_search_attributes_spend_to_the_calling_key(monkeypatch): ) +def _perplexity_router() -> MagicMock: + router = MagicMock() + router.search_tools = [ + { + "search_tool_name": "perplexity-sonar-pro", + "litellm_params": {"search_provider": "perplexity", "api_key": "fake-key"}, + } + ] + return router + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "parent_kwargs", + [ + pytest.param( + { + "litellm_call_id": "parent-call-1", + "litellm_trace_id": "trace-abc", + "litellm_session_id": "session-abc", + "metadata": { + "user_api_key_auth": UserAPIKeyAuth(api_key="hashed-sk-1234"), + "session_id": "session-abc", + }, + }, + id="chat-completions-call-kwargs", + ), + pytest.param( + { + "litellm_call_id": "parent-call-1", + "litellm_metadata": { + "user_api_key_auth": UserAPIKeyAuth(api_key="hashed-sk-1234"), + "session_id": "session-abc", + "trace_id": "trace-abc", + }, + }, + id="anthropic-messages-litellm-metadata", + ), + pytest.param( + { + "litellm_params": { + "litellm_call_id": "parent-call-1", + "litellm_trace_id": "trace-abc", + "metadata": {"user_api_key_auth": UserAPIKeyAuth(api_key="hashed-sk-1234")}, + "litellm_metadata": {"session_id": "session-abc"}, + } + }, + id="logging-payload-with-both-metadata-keys", + ), + ], +) +async def test_execute_search_inherits_parent_request_session_and_trace( + monkeypatch: pytest.MonkeyPatch, parent_kwargs: dict[str, object] +): + """The intercepted asearch is billed as its own call but must land in the parent request's + session and trace, otherwise every search shows up as a separate one-call session in SpendLogs.""" + import litellm + from litellm.proxy import proxy_server + from litellm.proxy.spend_tracking.spend_tracking_utils import _get_session_id_for_spend_log + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="perplexity-sonar-pro") + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + monkeypatch.setattr(proxy_server, "llm_router", _perplexity_router()) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + + await logger._execute_search("what is litellm", kwargs=parent_kwargs) + + forwarded = mock_asearch.await_args.kwargs + assert forwarded["litellm_session_id"] == "session-abc" + assert forwarded["litellm_trace_id"] == "trace-abc" + assert forwarded["litellm_metadata"]["session_id"] == "session-abc" + assert forwarded["litellm_metadata"]["trace_id"] == "trace-abc" + assert forwarded["litellm_metadata"]["parent_request_id"] == "parent-call-1" + assert forwarded["litellm_metadata"]["user_api_key"] == "hashed-sk-1234" + assert forwarded["litellm_metadata"]["model_group"] == "perplexity-sonar-pro" + assert "litellm_call_id" not in forwarded + assert ( + _get_session_id_for_spend_log( + kwargs={"litellm_trace_id": forwarded["litellm_trace_id"]}, + metadata=forwarded["litellm_metadata"], + standard_logging_payload=None, + omit_when_missing=True, + ) + == "session-abc" + ) + + +@pytest.mark.asyncio +async def test_execute_search_forwards_parent_otel_span_from_key_auth(monkeypatch: pytest.MonkeyPatch): + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="perplexity-sonar-pro") + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + monkeypatch.setattr(proxy_server, "llm_router", _perplexity_router()) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + parent_span = object() + + await logger._execute_search( + "what is litellm", + kwargs={"metadata": {"user_api_key_auth": UserAPIKeyAuth(api_key="sk", parent_otel_span=parent_span)}}, + ) + + assert mock_asearch.await_args.kwargs["litellm_metadata"]["litellm_parent_otel_span"] is parent_span + + +@pytest.mark.asyncio +async def test_execute_search_without_parent_session_does_not_invent_one(monkeypatch: pytest.MonkeyPatch): + """A parent request with no session/trace must not stamp empty correlation keys on the search.""" + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="perplexity-sonar-pro") + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + monkeypatch.setattr(proxy_server, "llm_router", _perplexity_router()) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + + await logger._execute_search( + "what is litellm", + kwargs={"litellm_params": {"metadata": {"user_api_key_auth": UserAPIKeyAuth(api_key="sk"), "prompt": "x"}}}, + ) + + forwarded = mock_asearch.await_args.kwargs + assert "litellm_session_id" not in forwarded + assert "litellm_trace_id" not in forwarded + assert not {"session_id", "trace_id", "parent_request_id", "prompt"} & forwarded["litellm_metadata"].keys() + + +@pytest.mark.asyncio +async def test_concurrent_searches_keep_their_own_parent_session(monkeypatch: pytest.MonkeyPatch): + import asyncio + + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="perplexity-sonar-pro") + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + monkeypatch.setattr(proxy_server, "llm_router", _perplexity_router()) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + + await asyncio.gather( + *( + logger._execute_search( + f"query {i}", + kwargs={"metadata": {"user_api_key_auth": UserAPIKeyAuth(api_key="sk"), "session_id": f"session-{i}"}}, + ) + for i in range(5) + ) + ) + + seen = { + call.kwargs["query"]: call.kwargs["litellm_metadata"]["session_id"] for call in mock_asearch.await_args_list + } + assert seen == {f"query {i}": f"session-{i}" for i in range(5)} + + @pytest.mark.asyncio async def test_execute_search_without_proxy_auth_context_stays_sdk_only(monkeypatch): """SDK callers have no key to attribute the search to, so no proxy metadata is invented.""" diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 4fe3d410ef8..76ccdec25d0 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1,5 +1,7 @@ +import json from collections.abc import Mapping from datetime import datetime, timezone +from typing import Final, cast import pytest @@ -297,42 +299,6 @@ def test_reasoning_tokens_gemini(_local_model_cost_map): ) -def test_reasoning_tokens_gemini_3_1_flash_lite(_local_model_cost_map): - """Test cost calculation for gemini-3.1-flash-lite-preview with reasoning tokens""" - model = "gemini-3.1-flash-lite-preview" - custom_llm_provider = "gemini" - - usage = Usage( - completion_tokens=1000, - prompt_tokens=500, - total_tokens=1500, - completion_tokens_details=CompletionTokensDetailsWrapper( - accepted_prediction_tokens=None, - audio_tokens=None, - reasoning_tokens=400, - rejected_prediction_tokens=None, - text_tokens=600, - ), - prompt_tokens_details=PromptTokensDetailsWrapper( - audio_tokens=None, cached_tokens=None, text_tokens=500, image_tokens=None - ), - ) - model_cost_map = litellm.model_cost[model] - prompt_cost, completion_cost = generic_cost_per_token( - model=model, - usage=usage, - custom_llm_provider=custom_llm_provider, - ) - - assert round(prompt_cost, 10) == round( - model_cost_map["input_cost_per_token"] * usage.prompt_tokens, - 10, - ) - assert round(completion_cost, 10) == round( - (model_cost_map["output_cost_per_token"] * usage.completion_tokens_details.text_tokens) - + (model_cost_map["output_cost_per_reasoning_token"] * usage.completion_tokens_details.reasoning_tokens), - 10, - ) def test_image_tokens_with_custom_pricing(): @@ -2219,65 +2185,8 @@ def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(_local_mo assert round(cost, 10) == round(expected_cost, 10) -def test_gemini_image_generation_cost_prefers_token_usage_metadata(_local_model_cost_map): - """ - When usage metadata exists on image responses, Gemini image generation cost - should be calculated from token pricing, not flat output_cost_per_image. - """ - - model = "gemini/gemini-3-pro-image-preview" - model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") - - input_text_tokens = 20 - input_image_tokens = 1120 - output_image_tokens = 1120 - prompt_tokens = input_text_tokens + input_image_tokens - - image_response = ImageResponse( - data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")], - usage=ImageUsage( - input_tokens=prompt_tokens, - input_tokens_details=ImageUsageInputTokensDetails( - text_tokens=input_text_tokens, - image_tokens=input_image_tokens, - ), - output_tokens=output_image_tokens, - total_tokens=prompt_tokens + output_image_tokens, - ), - ) - - cost = gemini_image_generation_cost_calculator( - model=model, - image_response=image_response, - ) - - expected_prompt_cost = prompt_tokens * model_info["input_cost_per_token"] - expected_completion_cost = output_image_tokens * model_info["output_cost_per_image_token"] - expected_total_cost = expected_prompt_cost + expected_completion_cost - - assert round(cost, 10) == round(expected_total_cost, 10) - # Ensure this is not falling back to flat per-image pricing. - assert cost != len(image_response.data) * model_info["output_cost_per_image"] -def test_gemini_image_generation_cost_falls_back_to_flat_image_pricing(_local_model_cost_map): - """ - Without usage metadata, Gemini image generation cost should fall back to - output_cost_per_image * number_of_images. - """ - - model = "gemini/gemini-3-pro-image-preview" - model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") - - image_response = ImageResponse(data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")]) - - cost = gemini_image_generation_cost_calculator( - model=model, - image_response=image_response, - ) - - expected_cost = len(image_response.data) * model_info["output_cost_per_image"] - assert round(cost, 10) == round(expected_cost, 10) def test_query_count_is_free_without_a_per_query_price(_local_model_cost_map): @@ -2458,23 +2367,6 @@ def test_vertex_global_or_absent_location_no_uplift(vertex_location, _local_mode assert base == located -@pytest.mark.parametrize("model", ["claude-opus-4-1", "gemini-2.0-flash-001"]) -def test_vertex_location_no_uplift_for_uniformly_priced_model(model, _local_model_cost_map): - """Models Google prices uniformly across endpoints (Gemini 2.x, Claude Opus 4.1 - and older) carry no multiplier and must not move with the location.""" - from litellm.types.utils import Usage - - usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) - - base = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="vertex_ai") - regional = generic_cost_per_token( - model=model, - usage=usage, - custom_llm_provider="vertex_ai", - vertex_location="us-east5", - ) - - assert base == regional, f"{model} should not have a regional-endpoint uplift" def test_vertex_uplift_invalid_multiplier_defaults_to_one(): @@ -3634,3 +3526,238 @@ def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_ assert creation == pytest.approx(expected_creation) assert creation_1h == pytest.approx(expected_creation_1h) + + +def _image_response(num_images: int = 1, usage: ImageUsage | None = None) -> ImageResponse: + return ImageResponse( + data=[ImageObject(url="https://example.com/img.png") for _ in range(num_images)], + usage=usage, + ) + + +_GPT_IMAGE_2_HIGH_1024: Final = {"quality": "high", "image_size": {"width": 1024, "height": 1024}} + + +@pytest.mark.parametrize( + ("model", "optional_params", "model_info", "num_images", "expected_cost"), + [ + ("fal-ai/unlisted-image-model", None, {"output_cost_per_image": 0.08}, 1, 0.08), + ("fal-ai/unlisted-image-model", None, {"output_cost_per_image": 0.08}, 2, 0.16), + ("fal-ai/unlisted-image-model", None, {"output_cost_per_image": "0.08"}, 1, 0.08), + ("openai/gpt-image-2", _GPT_IMAGE_2_HIGH_1024, {"output_cost_per_image": 0.5}, 1, 0.5), + ("openai/gpt-image-2", _GPT_IMAGE_2_HIGH_1024, {"mode": "image_generation"}, 1, 0.211), + ("openai/gpt-image-2", _GPT_IMAGE_2_HIGH_1024, {"output_cost_per_image": "0.08 USD"}, 1, 0.211), + ], +) +def test_route_image_generation_cost_honors_deployment_model_info( + _local_model_cost_map: None, + model: str, + optional_params: dict[str, object] | None, + model_info: ModelInfo, + num_images: int, + expected_cost: float, +) -> None: + cost = CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, + completion_response=_image_response(num_images), + custom_llm_provider="fal_ai", + optional_params=optional_params, + call_type="image_generation", + model_info=model_info, + ) + + assert cost == pytest.approx(expected_cost) + + +def test_route_image_generation_cost_openai_honors_deployment_input_cost_per_image( + _local_model_cost_map: None, +) -> None: + cost = CostCalculatorUtils.route_image_generation_cost_calculator( + model="dall-e-3", + completion_response=_image_response(), + custom_llm_provider="openai", + quality="standard", + size="1024-x-1024", + call_type="image_generation", + model_info={"input_cost_per_image": 0.07}, + ) + + assert cost == pytest.approx(0.07) + + + + +@pytest.mark.parametrize( + ("custom_llm_provider", "model"), + [ + ("gemini", "gemini/unlisted-image-model"), + ("vertex_ai", "vertex_ai/unlisted-image-model"), + ("azure_ai", "unlisted-image-model"), + ("openai", "gpt-image-unlisted"), + ], +) +def test_route_image_generation_cost_bills_deployment_image_price_when_unlisted_model_reports_tokens( + _local_model_cost_map: None, + custom_llm_provider: str, + model: str, +) -> None: + usage = ImageUsage( + input_tokens=10, + input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=10), + output_tokens=1290, + total_tokens=1300, + ) + + cost = CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, + completion_response=_image_response(num_images=2, usage=usage), + custom_llm_provider=custom_llm_provider, + call_type="image_generation", + model_info={"output_cost_per_image": 0.05}, + ) + + assert cost == pytest.approx(0.10) + + +def _batch_rates_model_info(**rates: object) -> ModelInfo: + return cast(ModelInfo, dict(rates)) + + +def test_get_batch_cost_rates_parses_string_rates(): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches="1e-06", + output_cost_per_token_batches="4e-06", + cache_read_input_token_cost_batches="1e-07", + input_cost_per_token_above_272k_tokens_batches="2e-06", + output_cost_per_token_above_272k_tokens_batches="6e-06", + cache_read_input_token_cost_above_272k_tokens_batches="2e-07", + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert (rates.input, rates.output, rates.cache_read) == (2e-6, 6e-6, 2e-7) + + +def test_get_batch_cost_rates_falls_back_to_the_flat_rates_when_tier_rates_are_unparsable(): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + cache_read_input_token_cost_batches=1e-7, + input_cost_per_token_above_272k_tokens_batches="two", + output_cost_per_token_above_272k_tokens_batches="six", + cache_read_input_token_cost_above_272k_tokens_batches="none", + cache_creation_input_token_cost_batches=1.25e-7, + cache_creation_input_token_cost_above_272k_tokens_batches="nope", + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert (rates.input, rates.output, rates.cache_read, rates.cache_creation) == (1e-6, 4e-6, 1e-7, 1.25e-7) + + +def test_get_batch_cost_rates_has_no_cached_rate_without_a_cached_batch_key(): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + cache_read_input_token_cost=2e-7, + input_cost_per_token_above_272k_tokens_batches=2e-6, + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert (rates.input, rates.output, rates.cache_read) == (2e-6, 4e-6, None) + + +@pytest.mark.parametrize(("prompt_tokens", "expected"), [(1_000, 1.25e-7), (272_000, 1.25e-7), (300_000, 2.5e-7)]) +def test_get_batch_cost_rates_reads_the_batch_cache_write_rate_for_the_crossed_tier(prompt_tokens, expected): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-7, + input_cost_per_token_above_272k_tokens_batches=2e-7, + cache_creation_input_token_cost_batches=1.25e-7, + cache_creation_input_token_cost_above_272k_tokens_batches=2.5e-7, + ), + Usage(prompt_tokens=prompt_tokens, completion_tokens=1, total_tokens=prompt_tokens + 1), + "openai", + ) + + assert rates.cache_creation == expected + + +@pytest.mark.parametrize( + ("tier_key", "attribute"), + [ + ("output_cost_per_token_above_272k_tokens_batches", "output"), + ("cache_read_input_token_cost_above_272k_tokens_batches", "cache_read"), + ("cache_creation_input_token_cost_above_272k_tokens_batches", "cache_creation"), + ], +) +def test_get_batch_cost_rates_crosses_a_tier_declared_without_an_input_tier_key(tier_key, attribute): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + cache_read_input_token_cost_batches=1e-7, + cache_creation_input_token_cost_batches=1.25e-7, + **{tier_key: 9e-6}, + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert getattr(rates, attribute) == 9e-6 + assert rates.input == 1e-6 + + +@pytest.mark.parametrize( + ("prompt_tokens", "expected_input", "expected_output"), + [(200_000, 1e-6, 4e-6), (250_000, 1e-6, 5e-6), (300_000, 2e-6, 5e-6)], +) +def test_get_batch_cost_rates_crosses_each_components_own_tier(prompt_tokens, expected_input, expected_output): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-6, + input_cost_per_token_above_272k_tokens_batches=2e-6, + output_cost_per_token_batches=4e-6, + output_cost_per_token_above_200k_tokens_batches=5e-6, + ), + Usage(prompt_tokens=prompt_tokens, completion_tokens=1, total_tokens=prompt_tokens + 1), + "openai", + ) + + assert (rates.input, rates.output) == (expected_input, expected_output) + + +def test_get_batch_cost_rates_has_no_cache_write_rate_without_a_cache_write_batch_key(): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-7, + input_cost_per_token_above_272k_tokens_batches=2e-7, + cache_creation_input_token_cost=2.5e-7, + cache_creation_input_token_cost_above_272k_tokens=5e-7, + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert rates.cache_creation is None diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 7bae2eaa338..41a2d19b8ab 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -109,26 +109,6 @@ def test_get_cost_for_built_in_tools_file_search(): assert cost == 0.00 -def test_get_cost_for_anthropic_web_search(): - """ - Test that Anthropic web search cost is tracked when usage.server_tool_use.web_search_requests - is set. Use claude-3-7-sonnet-20250219 (has search_context_cost_per_query) and - custom_llm_provider=anthropic so get_cost_for_anthropic_web_search is invoked. - """ - from litellm.types.utils import ServerToolUse, Usage - - model = "claude-3-7-sonnet-20250219" - usage = Usage(server_tool_use=ServerToolUse(web_search_requests=1)) - cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( - model=model, - usage=usage, - response_object=None, - standard_built_in_tools_params=None, - custom_llm_provider="anthropic", - ) - assert cost > 0.0 - - def test_get_cost_for_anthropic_web_search_with_server_tool_use_dict(): """ Anthropic-compatible passthrough responses can construct Usage from a raw @@ -145,88 +125,6 @@ def test_get_cost_for_anthropic_web_search_with_server_tool_use_dict(): ) -def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_drops_server_tool_use(): - """ - Regression: on the Anthropic /v1/messages sync cost path the response is the raw - Anthropic dict while the reconstructed OpenAI-shape Usage drops server_tool_use. - The web-search fee must still be charged by reading the count off the raw dict, - and the passed-in Usage must not be mutated. - """ - from litellm.types.utils import Usage - - model = "claude-3-7-sonnet-20250219" - web_search_requests = 3 - raw_response = { - "id": "msg_1", - "type": "message", - "role": "assistant", - "model": model, - "content": [{"type": "text", "text": "hi"}], - "stop_reason": "end_turn", - "stop_sequence": None, - "usage": { - "input_tokens": 100, - "output_tokens": 50, - "server_tool_use": {"web_search_requests": web_search_requests}, - }, - } - usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - assert getattr(usage, "server_tool_use", None) is None - - cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( - model=model, - usage=usage, - response_object=raw_response, - custom_llm_provider="anthropic", - standard_built_in_tools_params=None, - ) - - per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][ - "search_context_size_medium" - ] - assert cost == per_query_cost * web_search_requests - assert cost > 0.0 - assert getattr(usage, "server_tool_use", None) is None - - -def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_is_none(): - """ - Regression: when a caller hands the cost tracker a raw Anthropic dict without a - parallel Usage object, the web-search fee must still be priced per request from - usage.server_tool_use.web_search_requests on the dict instead of falling back to - the flat search_context_size_medium tier. - """ - model = "claude-3-7-sonnet-20250219" - web_search_requests = 4 - raw_response = { - "id": "msg_1", - "type": "message", - "role": "assistant", - "model": model, - "content": [{"type": "text", "text": "hi"}], - "stop_reason": "end_turn", - "stop_sequence": None, - "usage": { - "input_tokens": 100, - "output_tokens": 50, - "server_tool_use": {"web_search_requests": web_search_requests}, - }, - } - - cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( - model=model, - usage=None, - response_object=raw_response, - custom_llm_provider="anthropic", - standard_built_in_tools_params=None, - ) - - per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][ - "search_context_size_medium" - ] - assert cost == per_query_cost * web_search_requests - - def test_anthropic_web_search_zero_requests_from_raw_response_charges_zero(): """ Regression: a raw Anthropic dict reporting zero web search requests must price @@ -287,27 +185,6 @@ def test_anthropic_response_usage_block_preserves_server_tool_use(): assert dumped_usage["server_tool_use"] == {"web_search_requests": 2} -@pytest.mark.parametrize( - "model", ["gemini/gemini-2.0-flash-001", "gemini-2.0-flash-001"] -) -def test_get_cost_for_gemini_web_search(model): - """ - Test that the cost for a web search is 0.00 when no response object is provided - """ - from litellm.types.utils import PromptTokensDetailsWrapper, Usage - - usage = Usage( - prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=1) - ) - cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( - model=model, - usage=usage, - response_object=None, - standard_built_in_tools_params=None, - ) - assert cost > 0.0 - - def test_completion_cost_includes_web_search_without_standard_built_in_tools_params(): """ Test that completion_cost includes web search cost even when diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py new file mode 100644 index 00000000000..0e453e3f5eb --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py @@ -0,0 +1,157 @@ +from collections.abc import Mapping +from typing import Final + +import pytest + +from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import ( + ZERO_COST_COUNTER_NAME, + diagnose_zero_cost, + used_pricing_keys, + zero_cost_warning, +) +from litellm.types.utils import CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, Usage + +PER_SECOND_ENTRY: Final = {"input_cost_per_second": 0.00042, "output_cost_per_second": 0.00042} +FREE_ENTRY: Final = {"input_cost_per_token": 0, "output_cost_per_token": 0, "cache_read_input_token_cost": 2e-08} +PRICED_ENTRY: Final = {"input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06} +TEXT_USAGE: Final = Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + + +def test_missing_pricing_key_names_every_rate_the_usage_needs() -> None: + diagnostic = diagnose_zero_cost( + usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=PER_SECOND_ENTRY, calculation_failed=False + ) + + assert diagnostic == { + "reason": "missing_pricing_key", + "pricing_model": "dep-1", + "missing_pricing_keys": ("input_cost_per_token", "output_cost_per_token"), + } + + +def test_only_the_absent_rate_is_reported() -> None: + diagnostic = diagnose_zero_cost( + usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry={"input_cost_per_token": 1e-06}, calculation_failed=False + ) + + assert diagnostic is not None + assert diagnostic["missing_pricing_keys"] == ("output_cost_per_token",) + + +def test_free_model_stays_silent() -> None: + assert ( + diagnose_zero_cost(usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=FREE_ENTRY, calculation_failed=False) + is None + ) + + +@pytest.mark.parametrize("calculation_failed", [False, True]) +def test_request_without_usage_stays_silent(calculation_failed: bool) -> None: + usage = Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0) + + assert ( + diagnose_zero_cost( + usage=usage, pricing_model="dep-1", pricing_entry=PER_SECOND_ENTRY, calculation_failed=calculation_failed + ) + is None + ) + + +@pytest.mark.parametrize( + "entry", + [ + {"litellm_provider": "openai", "mode": "chat", "supports_prompt_caching": True}, + {"tiered_pricing": [{"range": [0, 128000], "input_cost_per_token": 0, "output_cost_per_token": 0}]}, + {"tiered_pricing": "not a tier table", "litellm_provider": "openai"}, + ], +) +def test_entry_that_declares_no_rate_stays_silent(entry: Mapping[str, object]) -> None: + assert ( + diagnose_zero_cost(usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=entry, calculation_failed=False) + is None + ) + + +def test_tiered_rate_counts_as_a_declared_rate() -> None: + entry = {"tiered_pricing": [{"range": [0, 128000], "input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06}]} + + diagnostic = diagnose_zero_cost( + usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=entry, calculation_failed=False + ) + + assert diagnostic is not None + assert diagnostic["reason"] == "missing_pricing_key" + + +def test_priced_entry_that_still_prices_to_zero_is_pricing_not_applied() -> None: + diagnostic = diagnose_zero_cost( + usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=PRICED_ENTRY, calculation_failed=False + ) + + assert diagnostic == {"reason": "pricing_not_applied", "pricing_model": "dep-1", "missing_pricing_keys": ()} + + +def test_calculator_failure_on_a_priced_entry_is_cost_calculation_error() -> None: + diagnostic = diagnose_zero_cost( + usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=PRICED_ENTRY, calculation_failed=True + ) + + assert diagnostic == {"reason": "cost_calculation_error", "pricing_model": "dep-1", "missing_pricing_keys": ()} + + +def test_calculator_failure_on_a_free_entry_stays_silent() -> None: + assert ( + diagnose_zero_cost(usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=FREE_ENTRY, calculation_failed=True) + is None + ) + + +def test_calculator_failure_on_an_entry_that_declares_no_rate_stays_silent() -> None: + entry: Final = {"litellm_provider": "openai", "mode": "chat", "supports_prompt_caching": True} + assert ( + diagnose_zero_cost(usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=entry, calculation_failed=True) + is None + ) + + +def test_audio_tokens_need_the_audio_rates() -> None: + usage = Usage( + prompt_tokens=10, + completion_tokens=20, + total_tokens=30, + prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=10, text_tokens=0), + completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=5, text_tokens=15), + ) + + assert used_pricing_keys(usage) == ( + "input_cost_per_audio_token", + "output_cost_per_token", + "output_cost_per_audio_token", + ) + diagnostic = diagnose_zero_cost( + usage=usage, pricing_model="gemini-audio", pricing_entry=PRICED_ENTRY, calculation_failed=False + ) + assert diagnostic is not None + assert diagnostic["missing_pricing_keys"] == ("input_cost_per_audio_token", "output_cost_per_audio_token") + + +def test_warning_names_the_request_the_entry_the_missing_keys_and_the_counter() -> None: + diagnostic = diagnose_zero_cost( + usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=PER_SECOND_ENTRY, calculation_failed=False + ) + assert diagnostic is not None + + message = zero_cost_warning( + diagnostic, + model_group="per-second-priced-chat", + model="openai/gpt-5.4-nano", + custom_llm_provider="openai", + usage=TEXT_USAGE, + ) + + assert "model_group=per-second-priced-chat" in message + assert "model=openai/gpt-5.4-nano" in message + assert "provider=openai" in message + assert "prompt_tokens=10 completion_tokens=20" in message + assert "pricing entry 'dep-1' has no input_cost_per_token, output_cost_per_token" in message + assert f'{ZERO_COST_COUNTER_NAME}{{reason="missing_pricing_key"}}' in message diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index c67f72680a8..62cf5680266 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_any_messages_to_chat_completion_str_messages_conversion, hoist_images_from_tool_messages, is_encrypted_reasoning_block, + merge_consecutive_system_messages, responses_reasoning_items_from_thinking_blocks, split_concatenated_json_objects, strip_encrypted_reasoning_from_messages, @@ -1846,3 +1847,95 @@ class TestEncryptedReasoningReplay: strip_encrypted_reasoning_from_messages(messages) assert messages == before + + +class TestMergeConsecutiveSystemMessages: + def test_merges_each_run_of_string_system_messages_with_a_blank_line(self): + messages = [ + {"role": "system", "content": "You are terse.", "cache_control": {"type": "ephemeral"}}, + {"role": "system", "content": "Skills: none."}, + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi"}, + {"role": "system", "content": "Reminder A"}, + {"role": "system", "content": "Reminder B"}, + {"role": "user", "content": "Bye"}, + ] + + merged = merge_consecutive_system_messages(messages) + + assert merged == [ + {"role": "system", "content": "You are terse.\n\nSkills: none.", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi"}, + {"role": "system", "content": "Reminder A\n\nReminder B"}, + {"role": "user", "content": "Bye"}, + ] + + def test_merges_into_text_parts_when_any_system_content_is_a_list(self): + cached_part = {"type": "text", "text": "Skills: none.", "cache_control": {"type": "ephemeral"}} + messages = [ + {"role": "system", "content": "You are terse."}, + {"role": "system", "content": [cached_part, {"type": "text", "text": "Be brief."}]}, + {"role": "system", "content": "Answer in English."}, + {"role": "user", "content": "Hello"}, + ] + + merged = merge_consecutive_system_messages(messages) + + assert merged == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are terse."}, + cached_part, + {"type": "text", "text": "Be brief."}, + {"type": "text", "text": "Answer in English."}, + ], + }, + {"role": "user", "content": "Hello"}, + ] + assert merged[0]["content"][1] is cached_part + + @pytest.mark.parametrize( + "messages", + [ + [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hello"}], + [{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "Hi"}], + [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hello"}, + {"role": "system", "content": "Reminder"}, + ], + [], + ], + ids=["single-system", "no-system", "separated-systems", "empty"], + ) + def test_leaves_messages_without_consecutive_system_messages_untouched(self, messages): + before = copy.deepcopy(messages) + + merged = merge_consecutive_system_messages(messages) + + assert merged == before + assert [message is original for message, original in zip(merged, messages)] == [True] * len(messages) + + @pytest.mark.parametrize( + ("messages", "expected_content"), + [ + ([{"role": "system"}, {"role": "system", "content": "Skills: none."}], "Skills: none."), + ([{"role": "system", "content": "You are terse."}, {"role": "system"}], "You are terse."), + ( + [{"role": "system"}, {"role": "system", "content": [{"type": "text", "text": "Be brief."}]}], + [{"type": "text", "text": "Be brief."}], + ), + ], + ids=["missing-then-str", "str-then-missing", "missing-then-list"], + ) + def test_skips_system_messages_without_content_when_merging(self, messages, expected_content): + merged = merge_consecutive_system_messages([*messages, {"role": "user", "content": "Hello"}]) + + assert merged == [{"role": "system", "content": expected_content}, {"role": "user", "content": "Hello"}] + + def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(self): + merged = merge_consecutive_system_messages([{"role": "system"}, {"role": "system"}, {"role": "user", "content": "Hi"}]) + + assert merged == [{"role": "system"}, {"role": "user", "content": "Hi"}] diff --git a/tests/test_litellm/litellm_core_utils/test_agentic_followup_kwargs.py b/tests/test_litellm/litellm_core_utils/test_agentic_followup_kwargs.py new file mode 100644 index 00000000000..af0fbdcc35b --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_agentic_followup_kwargs.py @@ -0,0 +1,66 @@ +from collections.abc import Mapping +from typing import Final + +from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs + + +def _build( + *, + request_kwargs: dict[str, object], + patch_kwargs: dict[str, object], + request_params: set[str], + fingerprints: list[str] | None = None, +) -> Mapping[str, object]: + return build_agentic_followup_kwargs( + request_kwargs=request_kwargs, + patch_kwargs=patch_kwargs, + request_params=request_params, + depth=0, + max_loops=3, + fingerprints=fingerprints if fingerprints is not None else [], + fingerprint="fp", + ) + + +def test_followup_kwargs_never_repeat_a_request_param(): + """Neither source may re-add a key the caller already sends as a request param, or the follow-up call raises a duplicate keyword""" + followup: Final = _build( + request_kwargs={"prompt_cache_key": "thread-1", "api_base": "https://a"}, + patch_kwargs={"prompt_cache_key": "thread-1", "metadata": {"user": "u1"}}, + request_params={"prompt_cache_key", "model", "input"}, + ) + + assert followup.keys().isdisjoint({"prompt_cache_key", "model", "input"}) + assert followup["api_base"] == "https://a" + assert followup["metadata"] == {"user": "u1"} + + +def test_followup_kwargs_let_the_plan_override_the_request(): + followup: Final = _build( + request_kwargs={"api_base": "https://request", "timeout": 5}, + patch_kwargs={"api_base": "https://plan"}, + request_params=set(), + ) + + assert followup["api_base"] == "https://plan" + assert followup["timeout"] == 5 + + +def test_followup_kwargs_carry_the_loop_bookkeeping_without_touching_the_inputs(): + fingerprints: Final = ["earlier"] + request_kwargs: Final = {"_agentic_loop_depth": 0, "max_agentic_loops": 9} + patch_kwargs: Final = {"_agentic_loop_fingerprints": ["stale"]} + + followup: Final = _build( + request_kwargs=request_kwargs, + patch_kwargs=patch_kwargs, + request_params=set(), + fingerprints=fingerprints, + ) + + assert followup["_agentic_loop_depth"] == 1 + assert followup["max_agentic_loops"] == 3 + assert followup["_agentic_loop_fingerprints"] == ["earlier", "fp"] + assert fingerprints == ["earlier"] + assert request_kwargs == {"_agentic_loop_depth": 0, "max_agentic_loops": 9} + assert patch_kwargs == {"_agentic_loop_fingerprints": ["stale"]} diff --git a/tests/test_litellm/litellm_core_utils/test_bug_report.py b/tests/test_litellm/litellm_core_utils/test_bug_report.py new file mode 100644 index 00000000000..62d7090b960 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_bug_report.py @@ -0,0 +1,230 @@ +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import cast +from urllib.parse import parse_qs, unquote_plus, urlparse + +import httpx +import pytest + +import litellm +from litellm._version import version +from litellm.exceptions import APIConnectionError, BadRequestError, InternalServerError +from litellm.litellm_core_utils.bug_report import ( + DISABLE_ENV_VAR, + ISSUE_URL_BASE, + MAX_FRAMES, + MAX_URL_LENGTH, + allowlisted, + bug_report_enabled, + bug_report_issue_url, + bug_report_notice, + build_bug_report, + build_environment_report, + should_report_bug, + strip_bug_report_notice, +) +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + +def test_build_bug_report_keeps_only_litellm_frames(): + with pytest.raises(BadRequestError) as raised: + get_llm_provider(cast(str, None)) + report = build_bug_report(raised.value, surface="sdk") + + assert report.litellm_frames + assert all(frame.startswith("litellm/") for frame in report.litellm_frames) + assert all("test_bug_report.py" not in frame for frame in report.litellm_frames) + + +def test_issue_url_never_contains_the_exception_message(): + secret = "sk-abcdefghijklmnopqrstuvwxyz1234567890" + prompt = "my social security number is 123-45-6789" + report = build_bug_report(RuntimeError(f"{secret} {prompt}"), surface="sdk") + url = bug_report_issue_url(report) + query = parse_qs(urlparse(url).query) + + assert url.startswith(ISSUE_URL_BASE) + assert secret not in url and "123-45-6789" not in url and "social" not in url + assert query["title"] == ["[Bug]: RuntimeError in litellm"] + assert query["version"] == [version] + assert query["template"] == ["bug_report.yml"] + assert query["domain"] == ["Python SDK: the litellm package itself"] + assert query["deployment"] == ["pip / Python SDK"] + assert "Exception: `RuntimeError`" in query["description"][0] + assert "Python:" in query["description"][0] + + +def test_issue_url_drops_unknown_provider_and_call_type(): + report = build_bug_report( + ValueError("boom"), + surface="proxy", + custom_llm_provider="acme-internal-gateway", + ) + query = parse_qs(urlparse(bug_report_issue_url(report)).query) + + assert report.custom_llm_provider is None + assert "acme" not in bug_report_issue_url(report) + assert "Provider: unknown" in query["description"][0] + assert "Endpoint / call: unknown" in query["description"][0] + + +def test_allowlisted_only_passes_exact_members(): + allowed = frozenset({"/v1/chat/completions"}) + + assert allowlisted("/v1/chat/completions", allowed) == "/v1/chat/completions" + assert allowlisted("/v1/chat/completions/../../admin", allowed) is None + assert allowlisted(None, allowed) is None + assert allowlisted(["/v1/chat/completions"], allowed) is None + assert allowlisted({"provider": "openai"}, allowed) is None + + +def test_build_bug_report_survives_unhashable_provider_from_request_data(): + report = build_bug_report( + KeyError("missing"), + surface="proxy", + custom_llm_provider={"name": "openai"}, + ) + + assert report.custom_llm_provider is None + + +def test_frames_are_capped_and_url_is_bounded(): + namespace: dict[str, object] = {} + exec( + compile( + "def recurse(depth):\n if depth == 0:\n raise RuntimeError('deep')\n recurse(depth - 1)\n", + str(Path(litellm.__file__).with_name("fake_deep_module.py")), + "exec", + ), + namespace, + ) + recurse = cast(Callable[[int], None], namespace["recurse"]) + + with pytest.raises(RuntimeError) as raised: + recurse(200) + report = build_bug_report(raised.value, surface="proxy") + + assert len(report.litellm_frames) == MAX_FRAMES + assert all(frame.startswith("litellm/fake_deep_module.py:") for frame in report.litellm_frames) + assert len(bug_report_issue_url(report)) <= MAX_URL_LENGTH + + +def test_issue_url_builds_without_a_traceback(): + exc = RuntimeError("no traceback") + assert exc.__traceback__ is None + report = build_bug_report(exc, surface="proxy") + + assert report.litellm_frames == () + assert bug_report_issue_url(report).startswith(ISSUE_URL_BASE) + + +def test_bug_report_can_be_disabled(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv(DISABLE_ENV_VAR, "true") + + assert bug_report_enabled() is False + assert should_report_bug(RuntimeError("boom")) is False + + +@pytest.mark.parametrize( + "exc", + [ + InternalServerError(message="upstream 500", llm_provider="openai", model="gpt-4"), + APIConnectionError( + message="connection reset", + llm_provider="openai", + model="gpt-4", + request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), + ), + BadRequestError(message="bad input", llm_provider="openai", model="gpt-4"), + "not an exception", + ], +) +def test_should_report_bug_skips_provider_and_network_errors(exc: object): + assert should_report_bug(exc) is False + + +def test_should_report_bug_accepts_plain_python_errors(): + assert should_report_bug(KeyError("missing")) is True + + +def test_proxy_known_provider_uses_translation_domain(): + report = build_bug_report( + RuntimeError("proxy failure"), + surface="proxy", + call_type="/v1/chat/completions", + custom_llm_provider="openai", + ) + query = parse_qs(urlparse(bug_report_issue_url(report)).query) + + assert report.custom_llm_provider == "openai" + assert query["domain"] == ["LLM translation: a specific provider's request or response"] + assert "Endpoint / call: /v1/chat/completions" in query["description"][0] + + +def test_strip_bug_report_notice(): + report = build_bug_report(RuntimeError("boom"), surface="sdk") + notice = bug_report_notice(report) + + assert strip_bug_report_notice(f"boom\n\n{notice}") == "boom\n" + assert strip_bug_report_notice("boom") == "boom" + + +def test_issue_description_renders_stream_and_config_block(): + report = build_bug_report( + RuntimeError("boom"), + surface="proxy", + stream=True, + config_lines=("router_settings.routing_strategy = least-busy", "litellm_settings.drop_params = true"), + ) + description = parse_qs(urlparse(bug_report_issue_url(report)).query)["description"][0] + + assert "Stream: true\n" in description + assert "```\nrouter_settings.routing_strategy = least-busy\nlitellm_settings.drop_params = true\n```" in description + + +@pytest.mark.parametrize("stream", [None, "true", 1]) +def test_issue_description_omits_stream_unless_it_is_a_bool(stream: object): + report = build_bug_report(RuntimeError("boom"), surface="proxy", stream=stream) + + assert report.stream is None + assert "Stream:" not in unquote_plus(bug_report_issue_url(report)) + + +def test_oversized_config_is_trimmed_from_the_end_before_any_frame(): + with pytest.raises(BadRequestError) as raised: + get_llm_provider(cast(str, None)) + config_lines = tuple(f"general_settings.flag_{index:04d} = true" for index in range(400)) + report = build_bug_report(raised.value, surface="proxy", config_lines=config_lines) + description = parse_qs(urlparse(url := bug_report_issue_url(report)).query)["description"][0] + + assert len(url) <= MAX_URL_LENGTH + assert all(frame in description for frame in report.litellm_frames) + assert "general_settings.flag_0000 = true" in description + assert "general_settings.flag_0399 = true" not in description + + +def test_issue_url_carries_exactly_the_environment_report_fields(): + report = build_bug_report( + RuntimeError("boom"), + surface="proxy", + config_lines=("litellm_settings.drop_params = true",), + ) + environment = report.environment + query = parse_qs(urlparse(bug_report_issue_url(report)).query) + description = query["description"][0] + + assert environment == build_environment_report( + surface="proxy", config_lines=("litellm_settings.drop_params = true",) + ) + assert query["version"] == [environment.litellm_version] + assert f"Surface: {environment.surface}\n" in description + assert f"LiteLLM: {environment.litellm_version}\n" in description + assert f"Python: {environment.python_version}\n" in description + assert "\nlitellm_settings.drop_params = true\n" in description + assert query.get("deployment") == (None if environment.deployment is None else [environment.deployment]) + + +def test_sdk_environment_reports_the_pip_deployment(): + assert build_environment_report(surface="sdk").deployment == "pip / Python SDK" diff --git a/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py index 434daab6ab5..0d7f735e6e5 100644 --- a/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py +++ b/tests/test_litellm/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -343,6 +343,40 @@ async def test_dispatcher_runs_followup_with_incremented_depth_and_patched_messa assert logger.cleanup_calls == 1 +@pytest.mark.asyncio +async def test_dispatcher_followup_does_not_repeat_a_request_param_found_in_request_kwargs( + restore_callbacks, +): + """Request kwargs that repeat a request param must not crash the follow-up + with a duplicate keyword, whether or not the plan copies them too.""" + followup = _plain_model_response("done") + request_kwargs = {"temperature": 0.2, "api_base": "https://a"} + plan = AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch(messages=_patched_messages(), kwargs=dict(request_kwargs)), + ) + litellm.callbacks = [_GateOnlyLogger(plan=plan, tool_calls={"tool_calls": [{"id": "call_abc"}]})] + + acompletion_mock = AsyncMock(return_value=followup) + with patch.object(litellm, "acompletion", acompletion_mock): + result = await maybe_run_chat_completion_agentic_loop( + response=_tool_call_model_response(), + model="gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + optional_params={"temperature": 0.2}, + kwargs=dict(request_kwargs), + logging_obj=_LoggingStub(), + custom_llm_provider="openai", + stream=False, + ) + + assert result is followup + acompletion_mock.assert_awaited_once() + call_kwargs = acompletion_mock.await_args.kwargs + assert call_kwargs["temperature"] == 0.2 + assert call_kwargs["api_base"] == "https://a" + + @pytest.mark.asyncio async def test_dispatcher_raises_when_depth_reaches_max_agentic_loops( restore_callbacks, diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py index b2ad13c205e..6eeea271127 100644 --- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py +++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py @@ -6,6 +6,8 @@ import pytest from litellm.litellm_core_utils.core_helpers import ( _FINISH_REASON_MAP, + bind_budget_reservation_to_callbacks, + budget_reservation_from_metadata, drop_params_env_flag, drop_params_flag, get_or_create_metadata_bucket, @@ -13,7 +15,60 @@ from litellm.litellm_core_utils.core_helpers import ( normalize_drop_params, reconstruct_model_name, redact_nested_match_and_regex_keys, + unbind_budget_reservation_from_callbacks, ) +from litellm.proxy._types import UserAPIKeyAuth + + +class TestBudgetReservationBinding: + """The request-end release skips a reservation a cost callback has claimed, so the claim + must land on the one dict auth stamped, through whichever metadata field or auth object + carries it, and a failed call must be able to hand it back.""" + + @staticmethod + def _reservation() -> dict: + return {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + + @pytest.mark.parametrize("metadata_variable_name", ["metadata", "litellm_metadata"]) + def test_reservation_stamped_on_the_metadata_is_bound(self, metadata_variable_name: str): + reservation = self._reservation() + + bind_budget_reservation_to_callbacks({metadata_variable_name: {"user_api_key_budget_reservation": reservation}}) + + assert reservation["callback_bound"] is True + + def test_reservation_reachable_only_through_the_auth_object_is_bound(self): + reservation = self._reservation() + user_api_key_auth = UserAPIKeyAuth(token="hashed") + user_api_key_auth.budget_reservation = reservation + + bind_budget_reservation_to_callbacks({"metadata": {"user_api_key_auth": user_api_key_auth}}) + + assert reservation["callback_bound"] is True + + def test_reservation_reachable_only_through_a_dumped_auth_object_is_bound(self): + reservation = self._reservation() + + bind_budget_reservation_to_callbacks({"metadata": {"user_api_key_auth": {"budget_reservation": reservation}}}) + + assert reservation["callback_bound"] is True + + def test_unbind_hands_a_claimed_reservation_back(self): + reservation = self._reservation() + litellm_params = {"litellm_metadata": {"user_api_key_budget_reservation": reservation}} + bind_budget_reservation_to_callbacks(litellm_params) + + unbind_budget_reservation_from_callbacks(litellm_params) + + assert reservation["callback_bound"] is False + + def test_request_without_a_reservation_binds_nothing(self): + metadata = {"user_api_key_auth": UserAPIKeyAuth(token="hashed")} + + bind_budget_reservation_to_callbacks({"metadata": metadata, "litellm_metadata": None}) + + assert budget_reservation_from_metadata(metadata) is None + assert "user_api_key_budget_reservation" not in metadata class TestGetOrCreateMetadataBucket: @@ -267,6 +322,28 @@ class TestRedactNestedMatchAndRegexKeys: assert redact_nested_match_and_regex_keys(None) is None assert redact_nested_match_and_regex_keys("plain") == "plain" + def test_redacts_custom_keys_without_changing_default_keys(self): + payload = { + "keyword": "secret-keyword", + "snippet": "secret-snippet", + "match": "secret-match", + "regex": "secret-regex", + "nested": [{"keyword": "nested-keyword", "match": "nested-match"}], + } + + custom_keys = redact_nested_match_and_regex_keys(payload, keys=("keyword", "snippet")) + default_keys = redact_nested_match_and_regex_keys(payload) + + assert custom_keys["keyword"] == "[REDACTED]" + assert custom_keys["snippet"] == "[REDACTED]" + assert custom_keys["nested"][0]["keyword"] == "[REDACTED]" + assert custom_keys["match"] == "secret-match" + assert custom_keys["regex"] == "secret-regex" + assert default_keys["match"] == "[REDACTED]" + assert default_keys["regex"] == "[REDACTED]" + assert default_keys["keyword"] == "secret-keyword" + assert default_keys["snippet"] == "secret-snippet" + @pytest.mark.parametrize( "value, expected", diff --git a/tests/test_litellm/litellm_core_utils/test_decode_special_tokens.py b/tests/test_litellm/litellm_core_utils/test_decode_special_tokens.py index d1a5f78a859..dd2daef5484 100644 --- a/tests/test_litellm/litellm_core_utils/test_decode_special_tokens.py +++ b/tests/test_litellm/litellm_core_utils/test_decode_special_tokens.py @@ -1,23 +1,11 @@ -from tokenizers import AddedToken, Tokenizer -from tokenizers.models import WordLevel -from tokenizers.pre_tokenizers import Whitespace -from tokenizers.processors import TemplateProcessing - from litellm import decode, encode +from tokenizers import Tokenizer + +TOKENIZER_JSON = """{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":3,"content":"[BOS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}],"normalizer":null,"pre_tokenizer":{"type":"Whitespace"},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"[BOS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}}],"pair":[{"Sequence":{"id":"A","type_id":0}},{"Sequence":{"id":"B","type_id":1}}],"special_tokens":{"[BOS]":{"id":"[BOS]","ids":[3],"tokens":["[BOS]"]}}},"decoder":null,"model":{"type":"WordLevel","vocab":{"[UNK]":0,"Hello":1,"World":2},"unk_token":"[UNK]"}}""" def _create_custom_tokenizer(): - tokenizer = Tokenizer( - WordLevel({"[UNK]": 0, "Hello": 1, "World": 2}, unk_token="[UNK]") - ) - tokenizer.pre_tokenizer = Whitespace() - tokenizer.add_special_tokens([AddedToken("[BOS]", special=True)]) - bos_token_id = tokenizer.token_to_id("[BOS]") - assert bos_token_id is not None - tokenizer.post_processor = TemplateProcessing( - single="[BOS] $A", - special_tokens=[("[BOS]", bos_token_id)], - ) + tokenizer = Tokenizer.from_str(TOKENIZER_JSON) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index cfe7470fa76..5c5c2c9536b 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -3,8 +3,6 @@ import openai import pytest import litellm - - from litellm.litellm_core_utils.exception_mapping_utils import ( ExceptionCheckers, _get_body_error_code, @@ -974,6 +972,31 @@ def test_an_unmapped_exception_with_no_model_or_provider_is_a_connection_error(q assert "boom" in raised.value.message +def test_unmapped_sdk_exception_includes_bug_report_link(quiet_exception_mapping): + with pytest.raises(litellm.APIConnectionError) as raised: + exception_type( + model="my-model", + custom_llm_provider="minimax", + original_exception=ValueError("boom"), + ) + + assert "https://github.com/BerriAI/litellm/issues/new?" in str(raised.value) + assert "ValueError" in str(raised.value) + + +def test_unmapped_sdk_exception_bug_report_link_can_be_disabled(quiet_exception_mapping, monkeypatch): + monkeypatch.setenv("LITELLM_DISABLE_BUG_REPORT_LINK", "true") + + with pytest.raises(litellm.APIConnectionError) as raised: + exception_type( + model="my-model", + custom_llm_provider="minimax", + original_exception=ValueError("boom"), + ) + + assert "https://github.com/BerriAI/litellm/issues/new?" not in str(raised.value) + + def _raise_and_map(model: str | None, original_exception: Exception, custom_llm_provider: str | None) -> None: """Calls exception_type() from inside the except block, as litellm/main.py does, so traceback.format_exc() has a real stack.""" diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py index 25a12bebf9a..f70b52a7026 100644 --- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py +++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py @@ -517,7 +517,7 @@ def test_shipped_gemini_chat_baseline_keeps_reasoning_effort_on_unmapped_model(s drop_params=False, ) assert isinstance(optional_params, dict) - assert optional_params["thinkingConfig"]["thinkingBudget"] > 0 + assert optional_params["thinkingConfig"]["thinkingLevel"] == "medium" assert optional_params["thinkingConfig"]["includeThoughts"] is True @@ -979,10 +979,6 @@ def test_shipped_tool_search_rule_fills_mapped_claude_entries_without_flag(shipp assert "supports_tool_search" not in litellm.model_cost[key] assert litellm.get_model_info(model, custom_llm_provider=provider)["supports_tool_search"] is True - assert "supports_tool_search" not in litellm.model_cost["claude-opus-4-1"] - opus_4_1_info = litellm.get_model_info("claude-opus-4-1", custom_llm_provider="anthropic") - assert opus_4_1_info.get("supports_tool_search") is None - assert "supports_tool_search" not in litellm.model_cost["azure_ai/claude-opus-5"] azure_opus_5_info = litellm.get_model_info("claude-opus-5", custom_llm_provider="azure_ai") assert azure_opus_5_info.get("supports_tool_search") is None diff --git a/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py b/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py index a34bc2af59d..7fb45e1b092 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py +++ b/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py @@ -4,6 +4,8 @@ Tests for get_litellm_params and related helpers. Ensures backward compatibility after sparse kwargs extraction optimization. """ +from typing import Final + import pytest from litellm.litellm_core_utils.get_litellm_params import ( @@ -12,6 +14,10 @@ from litellm.litellm_core_utils.get_litellm_params import ( get_litellm_params, ) +NAMED_PRICE_PARAMS: Final = frozenset( + {"input_cost_per_token", "output_cost_per_token", "input_cost_per_second", "output_cost_per_second"} +) + class TestGetBaseModelFromLitellmCallMetadata: def test_none_metadata_returns_none(self): @@ -40,10 +46,27 @@ class TestGetLitellmParamsKwargsExtraction: """Verify that optional kwargs are correctly extracted via sparse extraction.""" def test_no_kwargs_omits_optional_keys(self): - """When no kwargs passed, optional keys should not be in result.""" + """When no kwargs passed, optional keys are absent; the named price params are present as None.""" result = get_litellm_params(api_key="test-key") - for key in _OPTIONAL_KWARGS_KEYS: + for key in _OPTIONAL_KWARGS_KEYS - NAMED_PRICE_PARAMS: assert key not in result + for key in NAMED_PRICE_PARAMS: + assert result[key] is None + + def test_custom_pricing_kwargs_are_extracted(self) -> None: + from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model + from litellm.types.router import CustomPricingLiteLLMParams + + assert set(CustomPricingLiteLLMParams.model_fields) <= _OPTIONAL_KWARGS_KEYS + + result = get_litellm_params(output_cost_per_image=0.08, input_cost_per_audio_token=1e-6) + assert result["output_cost_per_image"] == 0.08 + assert result["input_cost_per_audio_token"] == 1e-6 + assert use_custom_pricing_for_model(result) is True + + result_without_prices = get_litellm_params() + assert "output_cost_per_image" not in result_without_prices + assert use_custom_pricing_for_model(result_without_prices) is False def test_present_kwargs_are_extracted(self): result = get_litellm_params( @@ -67,6 +90,15 @@ class TestGetLitellmParamsKwargsExtraction: assert "s3_endpoint_url" not in result_without_s3_kwargs assert "s3_region_name" not in result_without_s3_kwargs + def test_s3_credential_kwargs_are_forwarded_for_s3_signing(self): + result = get_litellm_params(s3_access_key_id="s3-key", s3_secret_access_key="s3-secret") + assert result["s3_access_key_id"] == "s3-key" + assert result["s3_secret_access_key"] == "s3-secret" + + result_without_s3_kwargs = get_litellm_params() + assert "s3_access_key_id" not in result_without_s3_kwargs + assert "s3_secret_access_key" not in result_without_s3_kwargs + def test_subset_of_kwargs_only_includes_provided(self): """Only provided kwargs appear, others remain absent.""" result = get_litellm_params(azure_ad_token="token123") diff --git a/tests/test_litellm/litellm_core_utils/test_get_llm_provider_logic.py b/tests/test_litellm/litellm_core_utils/test_get_llm_provider_logic.py index 1ecef9ffff7..ed8438bb1de 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_llm_provider_logic.py +++ b/tests/test_litellm/litellm_core_utils/test_get_llm_provider_logic.py @@ -1,5 +1,6 @@ from typing import Final +import httpx import pytest import litellm @@ -8,6 +9,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import ( get_llm_provider, is_registered_custom_provider, ) +from litellm.llms.custom_httpx.http_handler import HTTPHandler CUSTOM_PROVIDER: Final = "test-onprem-llm" @@ -53,3 +55,26 @@ def test_get_llm_provider_still_rejects_unregistered_prefix(registered_custom_pr ) def test_is_registered_custom_provider(registered_custom_provider: str, candidate: str | None, expected: bool) -> None: assert is_registered_custom_provider(candidate) is expected + + +def test_get_llm_provider_leaves_fal_ai_api_base_unset_for_global_fallback() -> None: + _, provider, _, api_base = get_llm_provider(model="fal_ai/fal-ai/flux/schnell") + assert provider == "fal_ai" + assert api_base is None + + _, _, _, explicit = get_llm_provider(model="fal_ai/fal-ai/flux/schnell", api_base="http://edge.local/fal") + assert explicit == "http://edge.local/fal" + + +def test_image_generation_fal_ai_egresses_to_global_api_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "api_base", "http://gateway.local/fal") + monkeypatch.setenv("FAL_AI_API_KEY", "test") + seen: Final[list[httpx.URL]] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request.url) + return httpx.Response(200, json={"images": [{"url": "https://fal.media/a.png"}]}) + + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))) + litellm.image_generation(model="fal_ai/fal-ai/flux/schnell", prompt="a red kite", client=client) + assert str(seen[0]).startswith("http://gateway.local/fal") diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index 53fee36b3a8..00977d9c3ee 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -218,7 +218,6 @@ def test_shipped_backup_marks_claude_4_6_plus_adaptive_not_4_0(): assert backup[adaptive]["supports_adaptive_thinking"] is True, adaptive for non_adaptive in [ - "claude-opus-4-20250514", "us.anthropic.claude-opus-4-20250514-v1:0", "claude-opus-4-5", ]: @@ -448,6 +447,23 @@ async def test_refetch_records_the_blob_id_of_the_bytes_served_and_the_fetch_eta assert get_model_cost_map_provenance() == {"source_revision": git_blob_id(body), "etag": 'W/"abc123"'} +@pytest.mark.asyncio +async def test_loaded_catalog_snapshot_follows_the_fetched_map_and_ignores_later_registrations(monkeypatch): + import litellm + + edited = json.loads(_real_map_bytes()) + edited["gpt-5.4-mini"]["max_input_tokens"] = 777 + client, _ = _mock_client([httpx.Response(200, content=json.dumps(edited).encode())]) + + result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) + + assert isinstance(result, ModelCostMapReloaded) + monkeypatch.setattr(litellm, "model_cost", result.model_cost_map) + litellm.register_model({"gpt-5.4-mini": {"max_input_tokens": 2048}}, persist_across_reloads=False) + assert litellm.model_cost["gpt-5.4-mini"]["max_input_tokens"] == 2048 + assert GetModelCostMap.loaded_model_cost_map()["gpt-5.4-mini"]["max_input_tokens"] == 777 + + @pytest.mark.asyncio async def test_refetch_revision_follows_the_bytes_not_the_url(): edited = json.loads(_real_map_bytes()) diff --git a/tests/test_litellm/litellm_core_utils/test_image_handling.py b/tests/test_litellm/litellm_core_utils/test_image_handling.py index 8fa4bd6c14d..21e97e97357 100644 --- a/tests/test_litellm/litellm_core_utils/test_image_handling.py +++ b/tests/test_litellm/litellm_core_utils/test_image_handling.py @@ -427,7 +427,7 @@ async def test_async_inline_remote_media_cancels_the_other_fetches_when_one_fail _SSRF_VERDICTS = ( SSRFError( "URL targets a blocked address (10.0.0.8). If this is a legitimate internal service, " - "add the host to `user_url_allowed_hosts` in general_settings." + "add the host to `user_url_allowed_hosts` in litellm_settings." ), SSRFError("DNS resolution failed for 'internal.example': [Errno 8] nodename nor servname provided, or not known"), SSRFError("No addresses found for 'internal.example'"), diff --git a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py index fe965f75f8f..a9a6509f6dd 100644 --- a/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py +++ b/tests/test_litellm/litellm_core_utils/test_initialize_dynamic_callback_params.py @@ -1,8 +1,11 @@ +from types import MappingProxyType +from typing import Final import pytest - +from pydantic import TypeAdapter from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + inherit_message_logging_privacy, initialize_standard_callback_dynamic_params, iter_client_callback_metadata_dicts, ) @@ -189,6 +192,20 @@ def test_empty_kwargs_returns_empty_params(): assert dict(params) == {} +@pytest.mark.parametrize("child_privacy", (False, True)) +def test_inherited_privacy_only_strengthens_child_and_resets(child_privacy: bool) -> None: + kwargs: Final = TypeAdapter(dict[str, object]).validate_python( + MappingProxyType({"turn_off_message_logging": child_privacy}) + ) + with inherit_message_logging_privacy(False): + assert initialize_standard_callback_dynamic_params(kwargs)["turn_off_message_logging"] is child_privacy + with inherit_message_logging_privacy(True), inherit_message_logging_privacy(False): + params: Final = initialize_standard_callback_dynamic_params(kwargs) + assert initialize_standard_callback_dynamic_params(kwargs)["turn_off_message_logging"] is child_privacy + assert params["turn_off_message_logging"] is True + assert initialize_standard_callback_dynamic_params().get("turn_off_message_logging") is None + + def test_newrelic_callback_params_are_not_extracted_from_request_kwargs(): kwargs = { "newrelic_api_key": "caller-key", @@ -228,9 +245,7 @@ def test_trusted_vars_overlay_uses_shared_parser_semantics(): # datadog handler consumes, so values are str()-coerced identically. from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD - params = initialize_standard_callback_dynamic_params( - {TRUSTED_CALLBACK_VARS_FIELD: {"newrelic_api_key": 12345}} - ) + params = initialize_standard_callback_dynamic_params({TRUSTED_CALLBACK_VARS_FIELD: {"newrelic_api_key": 12345}}) assert params.get("newrelic_api_key") == "12345" @@ -248,3 +263,19 @@ def test_validate_langfuse_environment_value(): for bad in ["Production", "langfuse-eu", "", "team a"]: with pytest.raises(ValueError, match="langfuse_environment"): validate_langfuse_environment_value(bad) + + +def test_arize_sampling_rates_are_picked_up_from_metadata(): + kwargs = { + "litellm_params": { + "metadata": { + "arize_success_sampling_rate": "0.5", + "arize_error_sampling_rate": "0.1", + } + } + } + + params = initialize_standard_callback_dynamic_params(kwargs) + + assert params.get("arize_success_sampling_rate") == "0.5" + assert params.get("arize_error_sampling_rate") == "0.1" diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 277ae33a076..5bdc0e9f8b8 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -1,17 +1,19 @@ import asyncio import contextlib import datetime +import json +import logging import os import sys -from collections.abc import Callable +import time +from collections.abc import Callable, Iterator, Mapping +from types import MappingProxyType from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch -import pytest - -import time - import httpx +import pytest +from mcp.types import AudioContent, CallToolResult, ImageContent, TextContent from openai._legacy_response import HttpxBinaryResponseContent import litellm @@ -25,6 +27,7 @@ from litellm.litellm_core_utils.litellm_logging import ( set_callbacks, ) from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo +from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import ResponseAPIUsage, ResponseCompletedEvent, ResponsesAPIResponse from litellm.types.utils import ( CallTypes, @@ -47,6 +50,272 @@ def logging_obj(): ) +@pytest.mark.asyncio +async def test_async_post_mcp_tool_call_hook_preserves_and_returns_content(logging_obj): + from litellm.types.mcp import MCPPostCallResponseObject + + class RedactingLogger(CustomLogger): + async def async_post_mcp_tool_call_hook( + self, + kwargs: dict[str, object], + response_obj: MCPPostCallResponseObject, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> MCPPostCallResponseObject: + assert isinstance(response_obj.mcp_tool_call_response, list) + assert isinstance(response_obj.mcp_tool_call_response[0], TextContent) + response_obj.mcp_tool_call_response = [TextContent(type="text", text="[REDACTED]")] + return response_obj + + logging_obj.dynamic_success_callbacks = [RedactingLogger()] + result = CallToolResult(content=[TextContent(type="text", text="SECRET-1234")], isError=False) + + hooked_content = await logging_obj.async_post_mcp_tool_call_hook( + kwargs=logging_obj.model_call_details, + response_obj=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + ) + + assert hooked_content.content == [TextContent(type="text", text="[REDACTED]")] + + +@pytest.mark.asyncio +async def test_async_post_mcp_tool_call_hook_chains_every_callback(logging_obj): + from litellm.types.mcp import MCPPostCallResponseObject + + class ReplacingLogger(CustomLogger): + def __init__(self, old: str, new: str) -> None: + super().__init__() + self.old: Final = old + self.new: Final = new + self.seen: list[str] = [] # mutable-ok: test records what each callback observed + + async def async_post_mcp_tool_call_hook( + self, + kwargs: dict[str, object], + response_obj: MCPPostCallResponseObject, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> MCPPostCallResponseObject: + first = response_obj.mcp_tool_call_response[0] + assert isinstance(first, TextContent) + self.seen.append(first.text) + return MCPPostCallResponseObject( + mcp_tool_call_response=[TextContent(type="text", text=first.text.replace(self.old, self.new))], + hidden_params=response_obj.hidden_params, + ) + + first_logger: Final = ReplacingLogger("SECRET", "[S]") + second_logger: Final = ReplacingLogger("1234", "[N]") + logging_obj.dynamic_success_callbacks = [first_logger, second_logger] + result = CallToolResult(content=[TextContent(type="text", text="SECRET-1234")], isError=False) + + hooked_content = await logging_obj.async_post_mcp_tool_call_hook( + kwargs=logging_obj.model_call_details, + response_obj=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + ) + + assert first_logger.seen == ["SECRET-1234"] + assert second_logger.seen == ["[S]-1234"] + assert hooked_content.content == [TextContent(type="text", text="[S]-[N]")] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["replace", "inplace", "empty", "inplace_none", "replace_none"]) +@pytest.mark.parametrize("structured", [False, True]) +async def test_mcp_content_rewrite_never_returns_stale_structured_data(logging_obj, mode, structured): + from litellm.types.llms.base import HiddenParams + from litellm.types.mcp import MCPPostCallResponseObject + + class Redactor(CustomLogger): + async def async_post_mcp_tool_call_hook(self, kwargs, response_obj, start_time, end_time): + block = response_obj.mcp_tool_call_response[0] + assert isinstance(block, TextContent) + if mode in ("inplace", "inplace_none"): + block.text = "[REDACTED]" + return None if mode == "inplace_none" else response_obj + if mode == "replace_none": + response_obj.mcp_tool_call_response = [TextContent(type="text", text="[REDACTED]")] + return None + return MCPPostCallResponseObject( + mcp_tool_call_response=[] if mode == "empty" else [TextContent(type="text", text="[REDACTED]")], + hidden_params=HiddenParams(response_cost=0.25), + ) + + logging_obj.dynamic_success_callbacks = [Redactor()] + result = CallToolResult( + content=[TextContent(type="text", text="SECRET-1234")], + structured_content={"nested": {"secret": "SECRET-1234"}} if structured else None, + meta={"request": "trace-1"}, + ) + logging_obj.model_call_details["original_response"] = result + returned = await logging_obj.async_post_mcp_tool_call_hook( + kwargs=logging_obj.model_call_details, + response_obj=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + ) + assert "SECRET-1234" not in result.model_dump_json(by_alias=True) + assert returned is result + assert result.content == ([] if mode == "empty" else [TextContent(type="text", text="[REDACTED]")]) + assert result.structured_content is None + assert result.is_error is structured + assert result.meta == {"request": "trace-1"} + assert logging_obj.model_call_details["original_response"] is result + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["none", "cost", "direct", "block", "exception"]) +async def test_mcp_callbacks_preserve_effective_result_and_cost(logging_obj, mode): + from litellm.types.llms.base import HiddenParams + from litellm.types.mcp import MCPPostCallResponseObject + + class Callback(CustomLogger): + async def async_post_mcp_tool_call_hook(self, kwargs, response_obj, start_time, end_time): + if mode == "exception": + response_obj.mcp_tool_call_response[0].text = "discarded" + raise ValueError("non-blocking callback") + if mode in ("direct", "block"): + original = kwargs["original_response"] + original.content = [TextContent(type="text", text="safe")] + original.structured_content = {"result": "safe"} + original.is_error = mode == "block" + if mode == "none" or mode == "direct": + return None + return MCPPostCallResponseObject( + mcp_tool_call_response=response_obj.mcp_tool_call_response, + hidden_params=HiddenParams(response_cost=0.25), + ) + + logging_obj.dynamic_success_callbacks = [Callback()] + result = CallToolResult( + content=[TextContent(type="text", text="original")], structured_content={"result": "original"} + ) + logging_obj.model_call_details["original_response"] = result + returned = await logging_obj.async_post_mcp_tool_call_hook( + kwargs=logging_obj.model_call_details, + response_obj=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + ) + expected = "safe" if mode in ("direct", "block") else "original" + assert result.content == [TextContent(type="text", text=expected)] + assert result.structured_content == {"result": expected} + assert result.is_error is (mode == "block") + assert returned is result + assert logging_obj.model_call_details.get("response_cost") == (0.25 if mode in ("cost", "block") else None) + + +@pytest.mark.asyncio +async def test_mcp_callback_cancellation_propagates_without_mutating_result(logging_obj): + class CancelledCallback(CustomLogger): + async def async_post_mcp_tool_call_hook(self, kwargs, response_obj, start_time, end_time): + response_obj.mcp_tool_call_response[0].text = "partial" + raise asyncio.CancelledError + + logging_obj.dynamic_success_callbacks = [CancelledCallback()] + result = CallToolResult(content=[TextContent(type="text", text="original")]) + with pytest.raises(asyncio.CancelledError): + await logging_obj.async_post_mcp_tool_call_hook( + kwargs={}, + response_obj=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + ) + assert result.content == [TextContent(type="text", text="original")] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("callbacks", [[], ["prometheus"]]) +@pytest.mark.parametrize("is_error", [False, True]) +async def test_mcp_without_custom_callbacks_preserves_mixed_content(logging_obj, callbacks, is_error): + logging_obj.dynamic_success_callbacks = callbacks + result = CallToolResult( + content=[ + TextContent(type="text", text="ok"), + ImageContent(type="image", data="aW1n", mime_type="image/png"), + AudioContent(type="audio", data="c291bmQ=", mime_type="audio/wav"), + ], + structured_content={"result": "ok"}, + is_error=is_error, + ) + before = result.model_dump() + returned = await logging_obj.async_post_mcp_tool_call_hook( + kwargs={}, + response_obj=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + ) + assert returned is result + assert returned.model_dump() == before + + +@pytest.mark.asyncio +@pytest.mark.parametrize("replace_structured", [False, True]) +@pytest.mark.parametrize("same_content", [False, True]) +async def test_mcp_native_structured_replacement_must_match_returned_content( + logging_obj, replace_structured, same_content +): + from litellm.types.mcp import MCPPostCallResponseObject + + class NativeReplacement(CustomLogger): + async def async_post_mcp_tool_call_hook(self, kwargs, response_obj, start_time, end_time): + original = kwargs["original_response"] + original.content[0].text = "native-safe" + if replace_structured: + original.structured_content["result"] = "native-safe" + return MCPPostCallResponseObject( + mcp_tool_call_response=[TextContent(type="text", text="native-safe" if same_content else "final-safe")], + hidden_params=response_obj.hidden_params, + ) + + result = CallToolResult( + content=[TextContent(type="text", text="SECRET-1234")], + structured_content={"result": "SECRET-1234"}, + ) + logging_obj.dynamic_success_callbacks = [NativeReplacement()] + returned = await logging_obj.async_post_mcp_tool_call_hook( + kwargs={"original_response": result}, response_obj=result, + start_time=datetime.datetime.now(), end_time=datetime.datetime.now(), + ) + assert returned is result + assert result.content == [TextContent(type="text", text="native-safe" if same_content else "final-safe")] + assert result.structured_content == ({"result": "native-safe"} if replace_structured and same_content else None) + assert result.is_error is not (replace_structured and same_content) + assert "SECRET-1234" not in result.model_dump_json() + + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["none", "wrapper", "exception"]) +@pytest.mark.parametrize("structured", [False, True]) +async def test_mcp_direct_content_edit_invalidates_stale_structured_data(logging_obj, mode, structured): + class DirectRedactor(CustomLogger): + async def async_post_mcp_tool_call_hook(self, kwargs, response_obj, start_time, end_time): + kwargs["original_response"].content[0].text = "[REDACTED]" + if mode == "exception": + raise ValueError("non-blocking callback after direct edit") + return response_obj if mode == "wrapper" else None + + result = CallToolResult( + content=[TextContent(type="text", text="SECRET-1234")], + structured_content={"result": "SECRET-1234"} if structured else None, + ) + logging_obj.dynamic_success_callbacks = [DirectRedactor()] + returned = await logging_obj.async_post_mcp_tool_call_hook( + kwargs={"original_response": result}, response_obj=result, + start_time=datetime.datetime.now(), end_time=datetime.datetime.now(), + ) + assert returned is result + assert result.content == [TextContent(type="text", text="[REDACTED]")] + assert result.structured_content is None + assert result.is_error is structured + assert "SECRET-1234" not in result.model_dump_json() + + def test_get_combined_callback_list_preserves_insertion_order(logging_obj): assert logging_obj.get_combined_callback_list( dynamic_success_callbacks=["prometheus", "langfuse", "datadog", "otel", "s3"], @@ -302,6 +571,417 @@ def test_response_cost_calculator_uses_router_model_id_from_litellm_metadata(): litellm.model_cost.pop(custom_model_id, None) +class TestZeroCostDiagnostic: + DEPLOYMENT_ID: Final = "lit7898-query-only-priced-deployment" + MODEL_GROUP: Final = "query-only-priced-chat" + QUERY_ONLY_PRICING: Final = {"input_cost_per_query": 0.00042} + PER_SECOND_PRICING: Final = {"input_cost_per_second": 0.00042, "output_cost_per_second": 0.00042} + FREE_PRICING: Final = {"input_cost_per_token": 0, "output_cost_per_token": 0} + + @pytest.fixture(params=["query_only", "free"]) + def deployment_pricing(self, request: pytest.FixtureRequest) -> Iterator[Mapping[str, float]]: + pricing: Final = self.QUERY_ONLY_PRICING if request.param == "query_only" else self.FREE_PRICING + litellm.register_model(model_cost={self.DEPLOYMENT_ID: pricing}, persist_across_reloads=False) + try: + yield pricing + finally: + litellm.model_cost.pop(self.DEPLOYMENT_ID, None) + + def _logging_obj( + self, + pricing: Mapping[str, object], + stream: bool = False, + model: str = "openai/gpt-5.4-nano", + call_type: str = "completion", + deployment_id: str | None = DEPLOYMENT_ID, + custom_llm_provider: str = "openai", + ) -> LitellmLogging: + logging_obj: Final = LitellmLogging( + model=model, + messages=[{"role": "user", "content": "Hi"}], + stream=stream, + call_type=call_type, + start_time=time.time(), + litellm_call_id="lit7898", + function_id="fn", + ) + self._route_to_deployment( + logging_obj, pricing, model=model, deployment_id=deployment_id, custom_llm_provider=custom_llm_provider + ) + return logging_obj + + def _route_to_deployment( + self, + logging_obj: LitellmLogging, + pricing: Mapping[str, object], + model: str = "openai/gpt-5.4-nano", + deployment_id: str | None = DEPLOYMENT_ID, + custom_llm_provider: str = "openai", + ) -> None: + model_info: Final = pricing if deployment_id is None else {"id": deployment_id, **pricing} + logging_obj.update_environment_variables( + model=model, + user="", + optional_params={}, + litellm_params={"metadata": {"model_group": self.MODEL_GROUP, "model_info": model_info}}, + custom_llm_provider=custom_llm_provider, + ) + + @staticmethod + def _response( + usage: litellm.Usage | None = None, model: str = "gpt-5.4-nano", **hidden_params: object + ) -> ModelResponse: + response: Final = ModelResponse( + model=model, + choices=[litellm.Choices(message=litellm.Message(role="assistant", content="hello"))], + usage=usage, + ) + response._hidden_params = {"custom_llm_provider": "openai", **hidden_params} + return response + + @staticmethod + def _zero_cost_warnings(caplog: pytest.LogCaptureFixture) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.name == "LiteLLM" and record.levelno == logging.WARNING and "priced at $0" in record.getMessage() + ] + + def _assert_flagged(self, logging_obj: LitellmLogging, caplog: pytest.LogCaptureFixture) -> None: + assert logging_obj.model_call_details["zero_cost_diagnostic"] == { + "reason": "missing_pricing_key", + "pricing_model": self.DEPLOYMENT_ID, + "missing_pricing_keys": ("input_cost_per_token", "output_cost_per_token"), + } + warnings: Final = self._zero_cost_warnings(caplog) + assert len(warnings) == 1 + assert f"model_group={self.MODEL_GROUP}" in warnings[0] + assert f"pricing entry '{self.DEPLOYMENT_ID}' has no input_cost_per_token, output_cost_per_token" in warnings[0] + + def test_zero_cost_with_a_missing_rate_warns_once_and_is_recorded( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + logging_obj: Final = self._logging_obj(deployment_pricing) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + first_cost: Final = logging_obj._response_cost_calculator(result=self._response(usage)) + second_cost: Final = logging_obj._response_cost_calculator(result=self._response(usage)) + + assert first_cost == 0.0 + assert second_cost == 0.0 + if deployment_pricing is self.FREE_PRICING: + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + return + self._assert_flagged(logging_obj, caplog) + + def test_usage_less_stream_chunk_does_not_hide_the_final_response_diagnostic( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + usage: Final = litellm.Usage(prompt_tokens=8, completion_tokens=2, total_tokens=10) + logging_obj: Final = self._logging_obj(deployment_pricing, stream=True) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + logging_obj._response_cost_calculator(result=self._response(usage=None)) + logging_obj._response_cost_calculator(result=self._response(usage)) + + if deployment_pricing is self.FREE_PRICING: + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + return + self._assert_flagged(logging_obj, caplog) + + def test_terminal_responses_stream_event_is_judged_by_its_inner_response( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + logging_obj: Final = self._logging_obj(deployment_pricing, stream=True, call_type="aresponses") + event: Final = ResponseCompletedEvent( + type="response.completed", + response=ResponsesAPIResponse( + id="resp-lit7898", + created_at=1, + object="response", + status="completed", + model="gpt-5.4-nano", + output=[], + usage=ResponseAPIUsage(input_tokens=10, output_tokens=20, total_tokens=30), + ), + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + cost: Final = logging_obj._response_cost_calculator(result=event) + + assert cost == 0.0 + if deployment_pricing is self.FREE_PRICING: + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + return + self._assert_flagged(logging_obj, caplog) + + def test_precomputed_zero_hidden_cost_is_flagged_and_lands_in_the_payload( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + logging_obj: Final = self._logging_obj(deployment_pricing) + response: Final = self._response(usage, response_cost=0.0, model_id=self.DEPLOYMENT_ID) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + logging_obj._process_hidden_params_and_response_cost( + response, start_time=datetime.datetime.now(), end_time=datetime.datetime.now() + ) + + payload: Final = logging_obj.model_call_details["standard_logging_object"] + assert payload["response_cost"] == 0.0 + if deployment_pricing is self.FREE_PRICING: + assert payload["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + return + self._assert_flagged(logging_obj, caplog) + assert payload["zero_cost_diagnostic"] == logging_obj.model_call_details["zero_cost_diagnostic"] + + def test_uncomputed_hidden_cost_is_not_a_zero_cost( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + logging_obj: Final = self._logging_obj(deployment_pricing) + response: Final = self._response(usage, response_cost=None, model_id=self.DEPLOYMENT_ID) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + logging_obj._process_hidden_params_and_response_cost( + response, start_time=datetime.datetime.now(), end_time=datetime.datetime.now() + ) + + assert logging_obj.model_call_details["standard_logging_object"]["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + + def test_unbilled_read_route_with_usage_stays_silent( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + logging_obj: Final = self._logging_obj(deployment_pricing, call_type="aget_responses") + response: Final = self._response(usage, response_cost=0.0, model_id=self.DEPLOYMENT_ID) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + logging_obj._process_hidden_params_and_response_cost( + response, start_time=datetime.datetime.now(), end_time=datetime.datetime.now() + ) + + assert logging_obj.model_call_details["standard_logging_object"]["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + + def test_unmapped_model_that_fails_cost_calculation_stays_silent(self, caplog: pytest.LogCaptureFixture) -> None: + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + logging_obj: Final = self._logging_obj( + {}, model="openai/lit7898-unmapped-model", deployment_id="lit7898-unmapped-deployment" + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + cost: Final = logging_obj._response_cost_calculator( + result=self._response(usage, model="lit7898-unmapped-model") + ) + + assert cost is None + assert logging_obj.model_call_details["response_cost_failure_debug_information"] is not None + assert logging_obj.model_call_details.get("zero_cost_diagnostic") is None + assert self._zero_cost_warnings(caplog) == [] + + def test_malformed_usage_never_raises_out_of_the_cost_calculator( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + logging_obj: Final = self._logging_obj(deployment_pricing) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + cost: Final = logging_obj._response_cost_calculator( + result={"model": "gpt-5.4-nano", "usage": {"prompt_tokens": "n/a", "completion_tokens": 3}} + ) + + assert cost is None + assert logging_obj.model_call_details.get("zero_cost_diagnostic") is None + assert self._zero_cost_warnings(caplog) == [] + + def test_usage_less_evaluation_between_two_zero_cost_findings_does_not_warn_twice( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + usage: Final = litellm.Usage(prompt_tokens=8, completion_tokens=2, total_tokens=10) + logging_obj: Final = self._logging_obj(deployment_pricing, stream=True, call_type="anthropic_messages") + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + logging_obj._response_cost_calculator(result=self._response(usage=None)) + logging_obj._response_cost_calculator(result=self._response(usage)) + logging_obj._response_cost_calculator(result=self._response(usage=None)) + logging_obj._response_cost_calculator(result=self._response(usage)) + + if deployment_pricing is self.FREE_PRICING: + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + return + self._assert_flagged(logging_obj, caplog) + + def test_retry_that_prices_clears_the_diagnostic_and_a_later_zero_cost_is_recorded_silently( + self, caplog: pytest.LogCaptureFixture + ) -> None: + priced_id: Final = "lit7898-priced-deployment" + priced_pricing: Final = {"input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06} + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + litellm.register_model( + model_cost={self.DEPLOYMENT_ID: self.QUERY_ONLY_PRICING, priced_id: priced_pricing}, + persist_across_reloads=False, + ) + try: + logging_obj: Final = self._logging_obj(self.QUERY_ONLY_PRICING) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + assert logging_obj._response_cost_calculator(result=self._response(usage)) == 0.0 + self._assert_flagged(logging_obj, caplog) + + self._route_to_deployment(logging_obj, priced_pricing, deployment_id=priced_id) + assert logging_obj._response_cost_calculator(result=self._response(usage)) == pytest.approx(5e-05) + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + + self._route_to_deployment(logging_obj, self.QUERY_ONLY_PRICING) + assert logging_obj._response_cost_calculator(result=self._response(usage)) == 0.0 + + assert logging_obj.model_call_details["zero_cost_diagnostic"]["reason"] == "missing_pricing_key" + assert len(self._zero_cost_warnings(caplog)) == 1 + finally: + litellm.model_cost.pop(self.DEPLOYMENT_ID, None) + litellm.model_cost.pop(priced_id, None) + + def test_one_request_evaluated_against_two_cost_map_entries_warns_once( + self, caplog: pytest.LogCaptureFixture + ) -> None: + dated_model: Final = "lit7898-nano-2026-03-17" + requested_model: Final = "lit7898-nano" + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + cost_map_entry: Final = {"litellm_provider": "openai", "mode": "chat", **self.QUERY_ONLY_PRICING} + litellm.register_model( + model_cost={dated_model: cost_map_entry, requested_model: cost_map_entry}, persist_across_reloads=False + ) + try: + logging_obj: Final = self._logging_obj( + {}, model=f"openai/{requested_model}", deployment_id="lit7898-cost-map-deployment" + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + assert logging_obj._response_cost_calculator(result=self._response(usage, model=dated_model)) == 0.0 + assert logging_obj._response_cost_calculator(result=self._response(usage, model=requested_model)) == 0.0 + + assert logging_obj.model_call_details["zero_cost_diagnostic"]["pricing_model"] == requested_model + warnings: Final = self._zero_cost_warnings(caplog) + assert len(warnings) == 1 + assert f"pricing entry '{dated_model}' has no input_cost_per_token, output_cost_per_token" in warnings[0] + finally: + litellm.model_cost.pop(dated_model, None) + litellm.model_cost.pop(requested_model, None) + + def test_free_deployment_without_a_router_id_is_judged_by_its_own_pricing( + self, caplog: pytest.LogCaptureFixture + ) -> None: + global_model: Final = "lit7898-priced-global" + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + litellm.register_model( + model_cost={ + global_model: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + } + }, + persist_across_reloads=False, + ) + try: + logging_obj: Final = self._logging_obj(self.FREE_PRICING, model=global_model, deployment_id=None) + response: Final = self._response(usage, model=global_model, response_cost=0.0) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + logging_obj._process_hidden_params_and_response_cost( + response, start_time=datetime.datetime.now(), end_time=datetime.datetime.now() + ) + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + finally: + litellm.model_cost.pop(global_model, None) + + def test_cache_hit_priced_for_saved_cost_stays_silent( + self, deployment_pricing: Mapping[str, float], caplog: pytest.LogCaptureFixture + ) -> None: + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + logging_obj: Final = self._logging_obj(deployment_pricing) + logging_obj.model_call_details["cache_hit"] = True + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + assert logging_obj._response_cost_calculator(result=self._response(usage), cache_hit=False) == 0.0 + + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + + def test_per_second_priced_deployment_bills_the_call_duration_and_stays_silent( + self, caplog: pytest.LogCaptureFixture + ) -> None: + per_second_id: Final = "lit8315-per-second-priced-deployment" + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + litellm.register_model(model_cost={per_second_id: self.PER_SECOND_PRICING}, persist_across_reloads=False) + try: + logging_obj: Final = self._logging_obj(self.PER_SECOND_PRICING, deployment_id=per_second_id) + response: Final = self._response(usage) + response._response_ms = 1000.0 + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + assert logging_obj._response_cost_calculator(result=response) == pytest.approx(0.00084) + + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert self._zero_cost_warnings(caplog) == [] + finally: + litellm.model_cost.pop(per_second_id, None) + + @pytest.mark.parametrize("spilled_over", [True, False]) + def test_ptu_deployment_is_judged_by_the_entry_the_calculator_priced_with( + self, spilled_over: bool, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + ) -> None: + router_model_id: Final = "lit7898-ptu-router-model-id" + served_model: Final = "azure/lit7898-ptu-served-model" + ptu_model_info: Final = { + "team_id": "team-1", + "ptu_count": 100, + "cost_per_ptu_per_hour": 1.0, + "ptu_effective_from": "2026-01-01", + **self.FREE_PRICING, + } + usage: Final = litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + litellm.register_model( + model_cost={ + router_model_id: {**self.FREE_PRICING, "litellm_provider": "azure", "mode": "chat"}, + served_model: {**self.QUERY_ONLY_PRICING, "litellm_provider": "azure", "mode": "chat"}, + }, + persist_across_reloads=False, + ) + monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True") + try: + logging_obj: Final = self._logging_obj( + ptu_model_info, model=served_model, deployment_id=router_model_id, custom_llm_provider="azure" + ) + spillover_headers: Final = {"llm_provider-x-ms-is-spilled-over": "true"} if spilled_over else {} + response: Final = self._response( + usage, model=served_model, custom_llm_provider="azure", additional_headers=spillover_headers + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + assert logging_obj._response_cost_calculator(result=response) == 0.0 + + warnings: Final = self._zero_cost_warnings(caplog) + if not spilled_over: + assert logging_obj.model_call_details["zero_cost_diagnostic"] is None + assert warnings == [] + return + assert logging_obj.model_call_details["zero_cost_diagnostic"] == { + "reason": "missing_pricing_key", + "pricing_model": served_model, + "missing_pricing_keys": ("input_cost_per_token", "output_cost_per_token"), + } + assert len(warnings) == 1 + assert f"pricing entry '{served_model}' has no input_cost_per_token, output_cost_per_token" in warnings[0] + finally: + litellm.model_cost.pop(router_model_id, None) + litellm.model_cost.pop(served_model, None) + + class TestGetRouterModelId: """Tests for the get_router_model_id helper method.""" @@ -407,7 +1087,6 @@ class TestGetRouterDeploymentModelInfo: logging_obj.litellm_params = {"api_base": ""} assert logging_obj.get_router_deployment_model_info() is None - def test_a_published_batch_rate_never_displaces_a_declared_standard_rate(self) -> None: """Ownership is per token direction, not per field. @@ -1111,7 +1790,9 @@ async def test_arealtime_marks_litellm_params_async(monkeypatch): @pytest.mark.asyncio -async def test_aresponses_websocket_hands_back_the_provider_failure_without_a_success_log(monkeypatch: pytest.MonkeyPatch): +async def test_aresponses_websocket_hands_back_the_provider_failure_without_a_success_log( + monkeypatch: pytest.MonkeyPatch, +): from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.responses.main import base_llm_http_handler @@ -2599,6 +3280,54 @@ def test_get_final_response_obj_with_empty_response_obj_and_list_init(): assert result[1].name == "Object2" +def test_get_final_response_obj_stores_the_text_a_post_call_guardrail_served(): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY + + raw = { + "id": "x", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Card: 4111 1111 1111 1111"}, + } + ], + } + + logged = StandardLoggingPayloadSetup.get_final_response_obj( + response_obj=raw, init_response_obj=raw, kwargs={SERVED_OUTPUT_TEXTS_KEY: ("Card: ",)} + ) + untouched = StandardLoggingPayloadSetup.get_final_response_obj(response_obj=raw, init_response_obj=raw, kwargs={}) + + assert isinstance(logged, dict) + assert logged["choices"][0]["message"]["content"] == "Card: " + assert logged["choices"][0]["finish_reason"] == "stop" + assert untouched == raw + + +def test_get_final_response_obj_redacts_the_served_text_when_message_logging_is_off(monkeypatch: pytest.MonkeyPatch): + import litellm + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY + + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + raw = { + "id": "x", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "Card: 4111"}}], + } + + logged = StandardLoggingPayloadSetup.get_final_response_obj( + response_obj=raw, + init_response_obj=raw, + kwargs={SERVED_OUTPUT_TEXTS_KEY: ("Card: ",), "litellm_params": {}}, + ) + + assert isinstance(logged, dict) + assert "" not in json.dumps(logged), logged + assert "4111" not in json.dumps(logged), logged + + def test_get_usage_as_dict(): """ Test get_usage_as_dict returns usage as plain dict from response_obj or combined_usage_object. @@ -3109,6 +3838,20 @@ def _make_logging_obj(stream: bool) -> LitellmLogging: ) +def test_get_response_ms_measures_a_float_start_time_against_a_datetime_end_time(): + """The files paths construct the logging object with ``time.time()`` while the success + handler stamps a datetime end, and the per-second cost path reads this window.""" + logging_obj = _make_logging_obj(stream=False) + logging_obj.update_environment_variables( + model="openai/codex-mini-latest", user="", optional_params={}, litellm_params={} + ) + start_seconds = logging_obj.model_call_details["start_time"] + assert isinstance(start_seconds, float) + logging_obj.model_call_details["end_time"] = datetime.datetime.fromtimestamp(start_seconds + 1.5) + + assert logging_obj.get_response_ms() == pytest.approx(1500) + + def test_get_assembled_streaming_response_returns_none_for_non_streaming(): """Non-streaming requests should return None so the streaming block is skipped.""" import datetime @@ -7001,6 +7744,141 @@ def test_passthrough_embeddings_result_swapped_for_callbacks(): assert swapped_result.data[0]["embedding"] == [0.1, 0.2, 0.3] +_PUBLISHED_BATCH_MODEL: Final = "lit-published-batch-tier-model" +_PUBLISHED_BATCH_DEPLOYMENT: Final = f"openai/{_PUBLISHED_BATCH_MODEL}" +_PUBLISHED_BATCH_RATES: Final = MappingProxyType( + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 2e-6, + "output_cost_per_token": 8e-6, + "input_cost_per_token_batches": 1.1e-6, + "output_cost_per_token_batches": 4.1e-6, + "cache_read_input_token_cost_batches": 1.2e-7, + "cache_creation_input_token_cost_batches": 1.3e-6, + "input_cost_per_token_above_272k_tokens_batches": 3.1e-6, + "output_cost_per_token_above_272k_tokens_batches": 7.1e-6, + "cache_read_input_token_cost_above_272k_tokens_batches": 3.2e-7, + "cache_creation_input_token_cost_above_272k_tokens_batches": 3.3e-6, + } +) +_PUBLISHED_INPUT_BATCH_KEYS: Final = ( + "input_cost_per_token_batches", + "input_cost_per_token_above_272k_tokens_batches", + "cache_read_input_token_cost_batches", + "cache_read_input_token_cost_above_272k_tokens_batches", + "cache_creation_input_token_cost_batches", + "cache_creation_input_token_cost_above_272k_tokens_batches", +) +_PUBLISHED_OUTPUT_BATCH_KEYS: Final = ( + "output_cost_per_token_batches", + "output_cost_per_token_above_272k_tokens_batches", +) + + +@pytest.fixture +def _published_batch_model(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.register_model( + model_cost={_PUBLISHED_BATCH_MODEL: {**_PUBLISHED_BATCH_RATES}}, persist_across_reloads=False + ) + + +def _batch_deployment_id(custom_pricing: dict[str, float]) -> str: + from litellm import Router + + router: Final = Router( + model_list=[ + { + "model_name": "published-batch", + "litellm_params": {"model": _PUBLISHED_BATCH_DEPLOYMENT, "api_key": "sk-test", **custom_pricing}, + } + ] + ) + return router.model_list[0]["model_info"]["id"] + + +def test_deployment_pricing_model_info_carries_every_published_input_batch_rate_when_only_output_is_declared( + _published_batch_model: None, +) -> None: + from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info + + info: Final = deployment_pricing_model_info( + _batch_deployment_id({"output_cost_per_token_batches": 4e-6}), _PUBLISHED_BATCH_DEPLOYMENT + ) + + assert info is not None + assert {key: info[key] for key in _PUBLISHED_INPUT_BATCH_KEYS} == { + key: _PUBLISHED_BATCH_RATES[key] for key in _PUBLISHED_INPUT_BATCH_KEYS + } + assert info["output_cost_per_token_batches"] == 4e-6 + assert info["output_cost_per_token_above_272k_tokens_batches"] is None + + +def test_deployment_pricing_model_info_carries_the_published_output_batch_tier_when_only_input_is_declared( + _published_batch_model: None, +) -> None: + from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info + + info: Final = deployment_pricing_model_info( + _batch_deployment_id({"input_cost_per_token_batches": 1e-6}), _PUBLISHED_BATCH_DEPLOYMENT + ) + + assert info is not None + assert {key: info[key] for key in _PUBLISHED_OUTPUT_BATCH_KEYS} == { + key: _PUBLISHED_BATCH_RATES[key] for key in _PUBLISHED_OUTPUT_BATCH_KEYS + } + assert info["input_cost_per_token_batches"] == 1e-6 + assert info["input_cost_per_token_above_272k_tokens_batches"] is None + assert info["cache_read_input_token_cost_batches"] is None + assert info["cache_creation_input_token_cost_batches"] is None + + +def test_batch_cost_calculator_bills_the_carried_output_tier_when_the_deployment_declares_its_own_input_rate( + _published_batch_model: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info + from litellm.types.utils import Usage + + info: Final = deployment_pricing_model_info( + _batch_deployment_id({"input_cost_per_token": 5e-6}), _PUBLISHED_BATCH_DEPLOYMENT + ) + assert info is not None + + prompt_cost, completion_cost = batch_cost_calculator( + usage=Usage(prompt_tokens=300_000, completion_tokens=10, total_tokens=300_010), + model=_PUBLISHED_BATCH_DEPLOYMENT, + custom_llm_provider="openai", + model_info=info, + ) + + assert prompt_cost == pytest.approx(300_000 * 5e-6 / 2) + assert completion_cost == pytest.approx( + 10 * _PUBLISHED_BATCH_RATES["output_cost_per_token_above_272k_tokens_batches"] + ) + + +def test_deployment_pricing_model_info_honors_a_tier_only_batch_override_over_the_published_flat_rates( + _published_batch_model: None, +) -> None: + from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info + + info: Final = deployment_pricing_model_info( + _batch_deployment_id({"input_cost_per_token_above_272k_tokens_batches": 1e-3}), _PUBLISHED_BATCH_DEPLOYMENT + ) + carried_keys: Final = tuple( + key + for key in (*_PUBLISHED_INPUT_BATCH_KEYS, *_PUBLISHED_OUTPUT_BATCH_KEYS) + if key != "input_cost_per_token_above_272k_tokens_batches" + ) + + assert info is not None + assert info["input_cost_per_token_above_272k_tokens_batches"] == 1e-3 + assert {key: info[key] for key in carried_keys} == {key: _PUBLISHED_BATCH_RATES[key] for key in carried_keys} + + def test_get_status_fields_ranks_guardrail_flagged_between_success_and_intervened(): """LIT-6894: a non-blocking flagged verdict must outrank success in the request-level guardrail_status but never mask an intervention.""" @@ -7066,22 +7944,41 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non return httpx.Response(200, json=mock_responses_api_response(content).model_dump()) if provider == "anthropic": - return httpx.Response(200, json={ - "id": "msg-audit", "type": "message", "role": "assistant", "model": "claude-haiku-4-5", - "content": [{"type": "text", "text": content}], "stop_reason": "end_turn", - "usage": {"input_tokens": 10, "output_tokens": 5}, - }) + return httpx.Response( + 200, + json={ + "id": "msg-audit", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": content}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + }, + ) if provider == "bedrock": - return httpx.Response(200, json={ - "output": {"message": {"role": "assistant", "content": [{"text": content}]}}, - "stopReason": "end_turn", "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15}, - "metrics": {"latencyMs": 1}, - }) - return httpx.Response(200, json={ - "id": "chatcmpl-audit", "object": "chat.completion", "created": 0, "model": "gpt-5.6", - "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, - }) + return httpx.Response( + 200, + json={ + "output": {"message": {"role": "assistant", "content": [{"text": content}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15}, + "metrics": {"latencyMs": 1}, + }, + ) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-audit", + "object": "chat.completion", + "created": 0, + "model": "gpt-5.6", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }, + ) async def capture(kwargs, response_obj, start_time, end_time): logs.put_nowait(kwargs["standard_logging_object"]) @@ -7092,11 +7989,15 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non handler.client = http_client client: Final = ( AsyncAzureOpenAI( - api_key="transport-only", azure_endpoint="https://azure.invalid", - api_version="2025-04-01-preview", http_client=http_client, + api_key="transport-only", + azure_endpoint="https://azure.invalid", + api_version="2025-04-01-preview", + http_client=http_client, ) - if provider == "azure" else AsyncOpenAI(api_key="transport-only", http_client=http_client) - if provider == "openai" else handler + if provider == "azure" + else AsyncOpenAI(api_key="transport-only", http_client=http_client) + if provider == "openai" + else handler ) model: Final = { "openai": "openai/gpt-5.6", @@ -7109,23 +8010,44 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non async def run(marker: str) -> None: if provider == "responses": await litellm.aresponses( - model=model, api_key="transport-only", client=client, max_output_tokens=128, - instructions="classifier-rubric", input=marker, + model=model, + api_key="transport-only", + client=client, + max_output_tokens=128, + instructions="classifier-rubric", + input=marker, metadata={"internal_call_origin": "autorouter_classifier"}, proxy_server_request={"body": {}, "originating_request_masked": {"input": f"source-only-{marker}"}}, - success_callback=[capture], num_retries=0, + success_callback=[capture], + num_retries=0, ) return await litellm.acompletion( - model=model, api_key="transport-only", client=client, max_tokens=128, - aws_access_key_id="transport-only", aws_secret_access_key="transport-only", aws_region_name="us-east-1", + model=model, + api_key="transport-only", + client=client, + max_tokens=128, + aws_access_key_id="transport-only", + aws_secret_access_key="transport-only", + aws_region_name="us-east-1", messages=[{"role": "system", "content": "classifier-rubric"}, {"role": "user", "content": marker}], metadata={"internal_call_origin": "autorouter_classifier"}, proxy_server_request={"body": {}, "originating_request_masked": {"input": f"source-only-{marker}"}}, - success_callback=[capture], num_retries=0, - **({"api_base": "https://azure.invalid", "api_version": "2025-04-01-preview"} if provider == "azure" else {}), - **({"extra_body": {"audit_context": "provider-extra"}, "extra_headers": {"X-Audit": "header-only-secret"}} - if provider in ("openai", "azure") else {}), + success_callback=[capture], + num_retries=0, + **( + {"api_base": "https://azure.invalid", "api_version": "2025-04-01-preview"} + if provider == "azure" + else {} + ), + **( + { + "extra_body": {"audit_context": "provider-extra"}, + "extra_headers": {"X-Audit": "header-only-secret"}, + } + if provider in ("openai", "azure") + else {} + ), ) await asyncio.gather(run("request-one"), run("request-two")) @@ -7149,14 +8071,17 @@ async def test_classifier_audit_matches_provider_transport(provider: str) -> Non @pytest.mark.parametrize("redaction", ["none", "global", "request", "header"]) @pytest.mark.parametrize("status", ["success", "failure"]) @pytest.mark.parametrize("call_type", ["completion", "acompletion", "responses", "aresponses"]) -def test_classifier_audit_obeys_message_logging_before_payload_emission(logging_obj, monkeypatch, redaction, status, call_type): +def test_classifier_audit_obeys_message_logging_before_payload_emission( + logging_obj, monkeypatch, redaction, status, call_type +): from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload monkeypatch.setattr(litellm, "turn_off_message_logging", redaction == "global") params: Final = { - "metadata": {"internal_call_origin": "autorouter_classifier", **( - {"headers": {"x-litellm-enable-message-redaction": "true"}} if redaction == "header" else {} - )}, + "metadata": { + "internal_call_origin": "autorouter_classifier", + **({"headers": {"x-litellm-enable-message-redaction": "true"}} if redaction == "header" else {}), + }, "proxy_server_request": {"body": {}, "originating_request_masked": {"input": "source-only"}}, } logging_obj.call_type = call_type @@ -7169,8 +8094,12 @@ def test_classifier_audit_obeys_message_logging_before_payload_emission(logging_ ) now: Final = datetime.datetime.now() payload: Final = get_standard_logging_object_payload( - kwargs={**logging_obj.model_call_details, "call_type": call_type}, init_response_obj={}, - start_time=now, end_time=now, logging_obj=logging_obj, status=status, + kwargs={**logging_obj.model_call_details, "call_type": call_type}, + init_response_obj={}, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status=status, ) assert payload is not None if redaction == "none": @@ -7421,7 +8350,13 @@ def _completed_responses_event(usage: ResponseAPIUsage) -> ResponseCompletedEven return ResponseCompletedEvent( type="response.completed", response=ResponsesAPIResponse( - id="resp-1", created_at=1, object="response", status="completed", model="codex-mini-latest", output=[], usage=usage + id="resp-1", + created_at=1, + object="response", + status="completed", + model="codex-mini-latest", + output=[], + usage=usage, ), ) @@ -7441,7 +8376,9 @@ def test_get_assembled_streaming_response_bills_a_provider_reported_usage_cost() now = datetime.datetime.now() assembled = logging_obj._get_assembled_streaming_response( - result=_completed_responses_event(ResponseAPIUsage(input_tokens=12, output_tokens=2, total_tokens=14, cost=0.0042)), + result=_completed_responses_event( + ResponseAPIUsage(input_tokens=12, output_tokens=2, total_tokens=14, cost=0.0042) + ), start_time=now, end_time=now, is_async=True, @@ -7452,18 +8389,40 @@ def test_get_assembled_streaming_response_bills_a_provider_reported_usage_cost() assert logging_obj._response_cost_calculator(result=assembled) == 0.0042 -def test_get_assembled_streaming_response_without_usage_cost_leaves_pricing_to_the_price_map(): - logging_obj = _responses_stream_logging_obj() - now = datetime.datetime.now() - assembled = logging_obj._get_assembled_streaming_response( - result=_completed_responses_event(ResponseAPIUsage(input_tokens=12, output_tokens=2, total_tokens=14)), - start_time=now, - end_time=now, - is_async=True, - streaming_chunks=[], + +def test_response_cost_calculator_prices_terminal_responses_event_from_its_response(): + logging_obj: Final = _responses_stream_logging_obj() + inner_response: Final = ResponsesAPIResponse( + id="resp-priced", + created_at=1, + object="response", + status="completed", + model="gpt-4o-mini", + output=[], + usage=ResponseAPIUsage(input_tokens=1840, output_tokens=412, total_tokens=2252), ) + event: Final = ResponseCompletedEvent(type="response.completed", response=inner_response) - assert "additional_headers" not in assembled._hidden_params - price_map_cost = logging_obj._response_cost_calculator(result=assembled) - assert price_map_cost is not None and 0 < price_map_cost != 0.0042 + event_cost: Final = logging_obj._response_cost_calculator(result=event) + inner_cost: Final = logging_obj._response_cost_calculator(result=inner_response) + + assert event_cost is not None and event_cost > 0 + assert event_cost == inner_cost + assert logging_obj.cost_breakdown["input_cost"] is not None and logging_obj.cost_breakdown["input_cost"] > 0 + + +class TestBudgetReservationBinding: + """The proxy builds a logging object for every route before calling anything, so a + logging object seeing the reservation is no promise that a cost callback will settle + it: the claim belongs to the call wrapper, and this object must leave it unbound.""" + + def test_update_environment_variables_leaves_the_reservation_unbound(self, logging_obj): + reservation: Final = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + + logging_obj.update_environment_variables( + litellm_params={"metadata": {"user_api_key_budget_reservation": reservation}}, optional_params={} + ) + + assert logging_obj.litellm_params["metadata"]["user_api_key_budget_reservation"] is reservation + assert reservation["callback_bound"] is False diff --git a/tests/test_litellm/litellm_core_utils/test_provider_affinity.py b/tests/test_litellm/litellm_core_utils/test_provider_affinity.py new file mode 100644 index 00000000000..edf4f5169b5 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_provider_affinity.py @@ -0,0 +1,107 @@ +import pytest + +from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY +from litellm.litellm_core_utils.provider_affinity import ( + add_provider_affinity_header, + get_stable_session_id, +) + + +@pytest.mark.parametrize( + ("litellm_params", "expected"), + [ + ({"litellm_session_id": "litellm-session"}, "litellm-session"), + ({"session_id": "direct-session"}, "direct-session"), + ({"metadata": {"session_id": "metadata-session"}}, "metadata-session"), + ({"litellm_metadata": {"session_id": "litellm-metadata-session"}}, "litellm-metadata-session"), + ], +) +def test_get_stable_session_id_uses_explicit_session_sources(litellm_params: dict, expected: str): + assert get_stable_session_id(litellm_params) == expected + + +def test_get_stable_session_id_does_not_use_trace_id(): + assert get_stable_session_id({"litellm_trace_id": "per-request-trace"}) is None + + +@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) +def test_get_stable_session_id_ignores_proxy_generated_session(metadata_key: str): + assert ( + get_stable_session_id( + { + "litellm_session_id": "generated-session", + metadata_key: { + "session_id": "generated-session", + SESSION_ID_GENERATED_METADATA_KEY: True, + }, + } + ) + is None + ) + + +def test_get_stable_session_id_prefers_explicit_session_over_proxy_generated_session(): + assert ( + get_stable_session_id( + { + "session_id": "explicit-session", + "litellm_session_id": "generated-session", + "metadata": { + "session_id": "generated-session", + SESSION_ID_GENERATED_METADATA_KEY: True, + }, + } + ) + == "explicit-session" + ) + + +def test_add_provider_affinity_header_maps_session_id(): + headers = add_provider_affinity_header( + headers={"Content-Type": "application/json"}, + litellm_params={ + "litellm_session_id": "session-123", + "provider_affinity_header": "X-Conversation-Id", + }, + ) + + assert headers == { + "Content-Type": "application/json", + "X-Conversation-Id": "session-123", + } + + +def test_add_provider_affinity_header_preserves_explicit_header_case_insensitively(): + headers = add_provider_affinity_header( + headers={"x-conversation-id": "explicit-session"}, + litellm_params={ + "litellm_session_id": "session-123", + "provider_affinity_header": "X-Conversation-Id", + }, + ) + + assert headers == {"x-conversation-id": "explicit-session"} + + +@pytest.mark.parametrize("session_id", ["session\r", "session\n", "session\0"]) +def test_add_provider_affinity_header_rejects_control_characters(session_id: str): + with pytest.raises(ValueError, match="session_id cannot contain HTTP header control characters"): + add_provider_affinity_header( + headers={}, + litellm_params={ + "litellm_session_id": session_id, + "provider_affinity_header": "X-Conversation-Id", + }, + ) + + +def test_add_provider_affinity_header_does_nothing_without_config_or_session(): + assert add_provider_affinity_header({}, {"litellm_session_id": "session-123"}) == {} + assert ( + add_provider_affinity_header( + {}, + {"provider_affinity_header": "X-Conversation-Id"}, + ) + == {} + ) + diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_errors.py b/tests/test_litellm/litellm_core_utils/test_realtime_errors.py index 1d2cf905f4e..999c660c286 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_errors.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_errors.py @@ -1,13 +1,17 @@ import json +from typing import cast import pytest from litellm.litellm_core_utils.realtime_errors import ( WEBSOCKET_CLOSE_REASON_MAX_BYTES, client_close_code, + close_after_upstream_handshake_refusal, realtime_error_event, + upstream_handshake_close_code, websocket_close_reason, ) +from litellm.types.realtime import RealtimeErrorEvent def test_realtime_error_event_shape(): @@ -52,3 +56,52 @@ def test_websocket_close_reason_truncates_multibyte_message_by_bytes(): ) def test_client_close_code_only_forwards_codes_a_server_may_send(upstream_code, expected): assert client_close_code(upstream_code) == expected + + +@pytest.mark.parametrize( + ("status_code", "expected"), + [(401, 1008), (403, 1008), (429, 1013), (500, 1011)], +) +def test_upstream_handshake_close_code_maps_http_status_to_close_code(status_code: int, expected: int): + assert upstream_handshake_close_code(status_code) == expected + + +class _RecordingWebSocket: + def __init__(self) -> None: + self.sent: list[str] = [] + self.closed: tuple[int, str | None] | None = None + + async def send_text(self, data: str) -> None: + self.sent.append(data) + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + self.closed = (code, reason) + + +@pytest.mark.asyncio +async def test_close_after_upstream_handshake_refusal_sends_error_event_then_policy_close(): + websocket = _RecordingWebSocket() + + await close_after_upstream_handshake_refusal(websocket, 401) + + assert len(websocket.sent) == 1 + event = cast(RealtimeErrorEvent, json.loads(websocket.sent[0])) + assert event["type"] == "error" + assert event["error"]["type"] == "server_error" + assert "401" in event["error"]["message"] + assert websocket.closed is not None + assert websocket.closed[0] == 1008 + assert websocket.closed[1] + + +@pytest.mark.asyncio +async def test_close_after_upstream_handshake_refusal_still_closes_when_send_fails(): + class _DeadWebSocket(_RecordingWebSocket): + async def send_text(self, data: str) -> None: + raise RuntimeError("socket gone") + + websocket = _DeadWebSocket() + + await close_after_upstream_handshake_refusal(websocket, 500) + + assert websocket.closed == (1011, "Upstream realtime handshake rejected with HTTP 500") diff --git a/tests/test_litellm/litellm_core_utils/test_redact_messages.py b/tests/test_litellm/litellm_core_utils/test_redact_messages.py index 276a67e0bd4..76d037ce760 100644 --- a/tests/test_litellm/litellm_core_utils/test_redact_messages.py +++ b/tests/test_litellm/litellm_core_utils/test_redact_messages.py @@ -304,6 +304,26 @@ class TestPerformRedaction: assert delta["thinking_blocks"] is None assert delta["audio"] is None + def test_redacts_text_completion_choices_in_standard_logging_object(self): + details = { + "standard_logging_object": { + "response": { + "object": "text_completion", + "choices": [ + {"text": " Paris.", "finish_reason": "stop", "index": 0}, + {"text": "\n\nBlue", "finish_reason": "length", "index": 1}, + ], + } + } + } + + perform_redaction(details, None) + + assert details["standard_logging_object"]["response"]["choices"] == [ + {"text": "redacted-by-litellm", "finish_reason": "stop", "index": 0}, + {"text": "redacted-by-litellm", "finish_reason": "length", "index": 1}, + ] + def test_redacts_object_choices_inside_model_response_dict(self): result = { "choices": [ @@ -1012,3 +1032,11 @@ def test_a_callback_that_redacts_itself_keeps_its_messages_but_not_the_classifie assert "classifier_input" not in stored assert stored["messages"] == payload["messages"] assert stored["response"] == payload["response"] + + +def test_perform_redaction_drops_the_served_output_texts_from_the_callback_kwargs() -> None: + from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY + + details: Final = {"litellm_params": {}, SERVED_OUTPUT_TEXTS_KEY: ("Card: ",)} + perform_redaction(details, None) + assert SERVED_OUTPUT_TEXTS_KEY not in details diff --git a/tests/test_litellm/litellm_core_utils/test_served_output_texts.py b/tests/test_litellm/litellm_core_utils/test_served_output_texts.py new file mode 100644 index 00000000000..ade39b8fca9 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_served_output_texts.py @@ -0,0 +1,114 @@ +from litellm.litellm_core_utils.served_output_texts import ( + SERVED_OUTPUT_TEXTS_KEY, + overlay_served_output_texts, + record_served_output_texts, + served_output_texts, + served_stream_output_texts, +) +from litellm.types.utils import Choices, Delta, Message, ModelResponse, ModelResponseStream, StreamingChoices + +RAW = "Card: 4111 1111 1111 1111" +MASKED = "Card: " + + +def _chat_response(*texts: str) -> ModelResponse: + return ModelResponse( + choices=[Choices(index=i, message=Message(content=text, role="assistant")) for i, text in enumerate(texts)] + ) + + +def _chat_dict(*texts: str) -> dict[str, object]: + return { + "id": "x", + "choices": [ + {"index": i, "finish_reason": "stop", "message": {"role": "assistant", "content": text}} + for i, text in enumerate(texts) + ], + } + + +def _choice_texts(response: object) -> tuple[str | None, ...]: + texts = served_output_texts(response) + assert texts is not None, response + return texts + + +def _stream_chunk(text: str, index: int = 0) -> ModelResponseStream: + return ModelResponseStream(choices=[StreamingChoices(index=index, delta=Delta(content=text))]) + + +def test_served_output_texts_reads_each_response_shape(): + assert served_output_texts(_chat_response(MASKED, "second")) == (MASKED, "second") + assert served_output_texts(_chat_dict(MASKED)) == (MASKED,) + assert served_output_texts( + { + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}], + } + ) == ("ab",) + responses_api: dict[str, object] = { + "object": "response", + "output": [ + {"type": "reasoning", "content": []}, + {"type": "message", "content": [{"type": "output_text", "text": MASKED}]}, + ], + } + assert served_output_texts(responses_api) == (MASKED,) + assert served_output_texts({"data": [{"embedding": [0.1]}]}) is None + assert served_output_texts("plain") is None + + +def test_served_stream_output_texts_joins_chat_chunks_and_reads_anthropic_sse(): + assert served_stream_output_texts([_stream_chunk("Card: "), _stream_chunk("")]) == (MASKED,) + sse = ( + 'event: message_start\ndata: {"type":"message_start","message":{"id":"m","type":"message","role":"assistant",' + '"content":[],"model":"x","usage":{"input_tokens":1,"output_tokens":1}}}\n\n', + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}\n\n', + f'event: content_block_delta\ndata: {{"type":"content_block_delta","index":0,"delta":{{"type":"text_delta","text":"{MASKED}"}}}}\n\n', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n', + 'event: message_stop\ndata: {"type":"message_stop"}\n\n', + ) + assert served_stream_output_texts(tuple(chunk.encode() for chunk in sse)) == (MASKED,) + assert served_stream_output_texts([]) is None + assert served_stream_output_texts([b"not sse"]) is None + + +def test_served_stream_output_texts_keeps_every_choice_index_when_chunks_carry_one_choice_each(): + chunks = [_stream_chunk("first ", 0), _stream_chunk(MASKED, 1), _stream_chunk("choice", 0)] + assert served_stream_output_texts(chunks) == ("first choice", MASKED) + + +def test_blanked_output_is_served_as_empty_text_and_overlaid(): + assert served_output_texts(_chat_response("")) == ("",) + assert served_output_texts({"type": "message", "role": "assistant", "content": [{"type": "text", "text": ""}]}) == ( + "", + ) + assert served_stream_output_texts([_stream_chunk("")]) == ("",) + assert _choice_texts(overlay_served_output_texts(_chat_dict(RAW), ("",))) == ("",) + + +def test_overlay_replaces_logged_choice_text_with_served_text(): + logged = _chat_dict(RAW, RAW) + overlaid = overlay_served_output_texts(logged, (MASKED,)) + assert _choice_texts(overlaid) == (MASKED, RAW) + assert isinstance(overlaid, dict) + assert overlaid["id"] == "x" + assert _choice_texts(logged) == (RAW, RAW) + + +def test_overlay_leaves_unreadable_inputs_untouched(): + logged = _chat_dict(RAW) + assert overlay_served_output_texts(logged, None) is logged + assert overlay_served_output_texts(logged, "not a tuple") is logged + assert overlay_served_output_texts(logged, (None,)) == logged + assert overlay_served_output_texts("text", (MASKED,)) == "text" + assert overlay_served_output_texts({"data": []}, (MASKED,)) == {"data": []} + + +def test_record_served_output_texts_only_writes_readable_texts(): + details: dict[str, object] = {} + record_served_output_texts(details, None) + assert SERVED_OUTPUT_TEXTS_KEY not in details + record_served_output_texts(details, (MASKED,)) + assert details[SERVED_OUTPUT_TEXTS_KEY] == (MASKED,) diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_server_tool_use.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_server_tool_use.py index 75508917a1e..e8a7ef74bfa 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_server_tool_use.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_server_tool_use.py @@ -99,29 +99,3 @@ def test_stream_chunk_builder_coerces_server_tool_use_to_pydantic(): assert server_tool_use.web_search_requests == 3 -def test_completion_cost_does_not_raise_on_streaming_web_search_response(): - """ - Regression: completion_cost(...) must not raise AttributeError when the - response was reconstructed by stream_chunk_builder from a streaming - Anthropic web_search call. - """ - chunks = [ - _make_text_chunk("hello"), - _make_finish_chunk_with_usage_dict_server_tool_use(), - ] - - rebuilt = stream_chunk_builder(chunks) - assert rebuilt is not None - - # The exact dollar amount depends on the model-pricing table; what matters - # for this regression is that it does NOT raise AttributeError on - # `dict has no attribute 'web_search_requests'`. - try: - cost = completion_cost(completion_response=rebuilt) - except AttributeError as e: # pragma: no cover - regression guard - pytest.fail( - "completion_cost raised AttributeError after stream_chunk_builder " - f"(issue #26153 regression): {e}" - ) - - assert isinstance(cost, (int, float)) diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 5d75c6699cf..af763da2d87 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1,5 +1,6 @@ from collections.abc import Mapping, Sequence from typing import Final +from unittest.mock import MagicMock import pytest @@ -7,6 +8,7 @@ import pytest from litellm import ChatCompletionUsageBlock, stream_chunk_builder from litellm.types.utils import GenericStreamingChunk from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor +from litellm.llms.anthropic.chat.handler import ModelResponseIterator from litellm.types.utils import ( ChatCompletionDeltaToolCall, ChatCompletionMessageToolCall, @@ -1650,6 +1652,71 @@ def test_calculate_usage_falls_back_to_prompt_counter_when_mock_stream_has_no_ad assert usage.prompt_tokens == 77 +@pytest.mark.parametrize( + ("message_delta_usage", "expected_cache_creation", "expected_cache_read"), + [ + ( + { + "input_tokens": 2, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 58352, + "output_tokens": 408, + }, + 0, + 58352, + ), + ({"output_tokens": 408}, 58352, 0), + ({"input_tokens": 2, "output_tokens": 408}, 58352, 0), + ], + ids=["delta_restates_cache_counts", "delta_reports_output_only", "delta_reports_input_and_output_only"], +) +def test_anthropic_stream_usage_takes_cache_counts_from_last_event_that_reports_them( + message_delta_usage: Mapping[str, int], expected_cache_creation: int, expected_cache_read: int +) -> None: + iterator: Final = ModelResponseIterator(streaming_response=MagicMock(), sync_stream=True, json_mode=False) + events: Final = ( + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [], + "stop_reason": None, + "usage": { + "input_tokens": 2, + "cache_creation_input_tokens": 58352, + "cache_read_input_tokens": 0, + "output_tokens": 1, + }, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hi"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": dict(message_delta_usage)}, + {"type": "message_stop"}, + ) + + response: Final = stream_chunk_builder( + chunks=[iterator.chunk_parser(event) for event in events], + messages=[{"role": "user", "content": "hi"}], + ) + + assert response.usage.cache_creation_input_tokens == expected_cache_creation + assert response.usage.cache_read_input_tokens == expected_cache_read + assert response.usage.prompt_tokens == 58354 + assert response.usage.prompt_tokens_details.cache_creation_tokens == expected_cache_creation + assert response.usage.prompt_tokens_details.cached_tokens == expected_cache_read + assert ( + response.usage.prompt_tokens + - response.usage.cache_read_input_tokens + - response.usage.cache_creation_input_tokens + == 2 + ) + + _ZERO_USAGE_TEXT_CHUNKS: Final = ( _openai_chunk(choices=[{"index": 0, "delta": {"role": "assistant", "content": "Hi"}, "finish_reason": None}]), _openai_chunk(choices=[{"index": 0, "delta": {"content": " there"}, "finish_reason": None}]), diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index f19a8891609..eccf44a1bda 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -633,9 +633,6 @@ def test_openai_token_with_image_and_text(): "model, base_model, input_tokens, user_max_tokens, expected_value", [ ("random-model", "random-model", 1024, 1024, 1024), - ("command", "command", 1000000, None, None), # model max = 4096 - ("command", "command", 4000, 256, 96), # model max = 4096 - ("command", "command", 4000, 10, 10), # model max = 4096 ("gpt-3.5-turbo", "gpt-3.5-turbo", 4000, 5000, 4096), # model max output = 4096 ], ) @@ -786,23 +783,23 @@ def test_token_counter(): import unittest -from litellm.utils import _select_tokenizer_helper, claude_json_str, encoding +from litellm.utils import _load_huggingface_tokenizer, _select_tokenizer_helper, claude_json_str, encoding # Clear the cache at module load to ensure clean state -_select_tokenizer_helper.cache_clear() +_load_huggingface_tokenizer.cache_clear() class TestTokenizerSelection(unittest.TestCase): def setUp(self): """Clear the LRU cache before each test method. - The _select_tokenizer_helper function is decorated with @lru_cache, - which can cause cache hits from previous tests when running with + The HuggingFace tokenizers behind _select_tokenizer_helper are cached with + @lru_cache, which can cause cache hits from previous tests when running with --dist=loadscope (tests from same file run on same worker). """ - _select_tokenizer_helper.cache_clear() + _load_huggingface_tokenizer.cache_clear() - @patch("litellm.utils.Tokenizer.from_pretrained") + @patch("litellm.utils.tokenizer_dispatch.from_pretrained") def test_llama3_tokenizer_api_failure(self, mock_from_pretrained): # Setup mock to raise an error mock_from_pretrained.side_effect = Exception("Failed to load tokenizer") @@ -817,7 +814,7 @@ class TestTokenizerSelection(unittest.TestCase): self.assertEqual(result["type"], "openai_tokenizer") self.assertEqual(result["tokenizer"], encoding) - @patch("litellm.utils.Tokenizer.from_pretrained") + @patch("litellm.utils.tokenizer_dispatch.from_pretrained") def test_cohere_tokenizer_api_failure(self, mock_from_pretrained): # Setup mock to raise an error mock_from_pretrained.side_effect = Exception("Failed to load tokenizer") @@ -837,10 +834,10 @@ class TestTokenizerSelection(unittest.TestCase): self.assertEqual(result["type"], "openai_tokenizer") self.assertEqual(result["tokenizer"], encoding) - @patch("litellm.utils.Tokenizer.from_str") - def test_claude_tokenizer_api_failure(self, mock_from_str): + @patch("litellm.utils.tokenizer_dispatch.anthropic") + def test_claude_tokenizer_api_failure(self, mock_anthropic): # Setup mock to raise an error - mock_from_str.side_effect = Exception("Failed to load tokenizer") + mock_anthropic.side_effect = Exception("Failed to load tokenizer") # Add Claude model to the list for testing litellm.anthropic_models = ["claude-2"] @@ -849,13 +846,13 @@ class TestTokenizerSelection(unittest.TestCase): result = _select_tokenizer_helper("claude-2") # Verify the attempt to load Claude tokenizer - mock_from_str.assert_called_once_with(claude_json_str) + mock_anthropic.assert_called_once_with() # Verify fallback to OpenAI tokenizer self.assertEqual(result["type"], "openai_tokenizer") self.assertEqual(result["tokenizer"], encoding) - @patch("litellm.utils.Tokenizer.from_pretrained") + @patch("litellm.utils.tokenizer_dispatch.from_pretrained") def test_llama2_tokenizer_api_failure(self, mock_from_pretrained): # Setup mock to raise an error mock_from_pretrained.side_effect = Exception("Failed to load tokenizer") diff --git a/tests/test_litellm/litellm_core_utils/test_tokenizer.py b/tests/test_litellm/litellm_core_utils/test_tokenizer.py new file mode 100644 index 00000000000..aa4a0fc6a1c --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_tokenizer.py @@ -0,0 +1,403 @@ +import copy +import os +import pickle +import subprocess +import sys +from pathlib import Path +from typing import Final, Literal + +import pytest +import tiktoken +from tokenizers import Tokenizer as ReferenceTokenizer + +import litellm +from litellm.caching._embedding_router import truncate_embedding_input +from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEncoding +from litellm.utils import claude_json_str +from tests.test_litellm.litellm_core_utils.test_decode_special_tokens import TOKENIZER_JSON + + +@pytest.mark.parametrize( + "name", ("cl100k_base", "o200k_base", "p50k_base", "p50k_edit", "r50k_base", "gpt2", "o200k_harmony") +) +@pytest.mark.parametrize( + "text", ("hello world", "café 漢字 🙂", "", "a\ud800b", "\ud83d\ude42", "🙂\ud83d\ude42\udfff", " " * 64) +) +def test_openai_encoding_matches_python_unicode_and_batches(name: str, text: str) -> None: + reference: Final = tiktoken.get_encoding(name) + encoding: Final = OpenAIEncoding.from_tiktoken(name) + expected: Final = reference.encode(text) + + assert encoding.encode(text) == expected + assert encoding.count(text) == len(expected) + assert encoding.encode_batch([text], num_threads=2) == reference.encode_batch([text], num_threads=2) + assert encoding.encode_ordinary_batch([text]) == reference.encode_ordinary_batch([text]) + assert encoding.decode_batch([expected]) == reference.decode_batch([expected]) + assert encoding.decode_bytes_batch([expected]) == reference.decode_bytes_batch([expected]) + + +@pytest.mark.parametrize("allowed", (frozenset(), frozenset({"<|endoftext|>"}), "all")) +@pytest.mark.parametrize("disallowed", (frozenset(), frozenset({"<|fim_prefix|>"}), "all")) +def test_openai_special_token_options_match_python( + allowed: frozenset[str] | Literal["all"], disallowed: frozenset[str] | Literal["all"] +) -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + encoding: Final = OpenAIEncoding.from_tiktoken(reference.name) + text: Final = "hello<|endoftext|><|fim_prefix|>world" + allowed_set: Final = reference.special_tokens_set if allowed == "all" else allowed + disallowed_set: Final = reference.special_tokens_set - allowed_set if disallowed == "all" else disallowed + if any(token in text for token in disallowed_set): + with pytest.raises(ValueError, match="disallowed special token"): + encoding.encode(text, allowed_special=allowed, disallowed_special=disallowed) + return + assert encoding.encode(text, allowed_special=allowed, disallowed_special=disallowed) == reference.encode( + text, allowed_special=allowed, disallowed_special=disallowed + ) + assert encoding.special_tokens_set == reference.special_tokens_set + assert encoding.eot_token == reference.eot_token + + +@pytest.mark.parametrize("errors", ("replace", "ignore", "backslashreplace", "strict")) +def test_openai_partial_token_decoding_preserves_error_policy(errors: str) -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + encoding: Final = OpenAIEncoding.from_tiktoken(reference.name) + tokens: Final = reference.encode("🙂")[:1] + assert encoding.decode_bytes(tokens) == reference.decode_bytes(tokens) + if errors == "strict": + with pytest.raises(UnicodeDecodeError): + encoding.decode(tokens, errors=errors) + return + assert encoding.decode(tokens, errors=errors) == reference.decode(tokens, errors=errors) + assert encoding.decode_tokens_bytes(tokens) == reference.decode_tokens_bytes(tokens) + + +def test_public_encoding_and_semantic_cache_preserve_truncated_unicode() -> None: + reference: Final = tiktoken.get_encoding(litellm.encoding.name) + text: Final = "🙂" + tokens: Final = reference.encode(text) + + assert litellm.encoding.encode(text, disallowed_special=()) == tokens + assert litellm.encoding.encode_batch([text]) == [tokens] + assert litellm.decode(tokens=tokens[:1]) == reference.decode(tokens[:1]) + assert truncate_embedding_input(text, "", 1) == reference.decode(tokens[:1]) + + +@pytest.mark.parametrize("add_special_tokens", (True, False)) +def test_huggingface_encoding_preserves_result_fields_and_serialization(add_special_tokens: bool) -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + tokenizer: Final = HuggingFaceTokenizer.from_str(TOKENIZER_JSON) + expected: Final = reference.encode("Hello World", add_special_tokens=add_special_tokens) + actual: Final = tokenizer.encode("Hello World", add_special_tokens=add_special_tokens) + + assert (actual.ids, actual.tokens, actual.type_ids, actual.offsets, actual.word_ids, actual.sequence_ids) == ( + expected.ids, + expected.tokens, + expected.type_ids, + expected.offsets, + expected.word_ids, + expected.sequence_ids, + ) + assert (actual.attention_mask, actual.special_tokens_mask, actual.n_sequences, len(actual)) == ( + expected.attention_mask, + expected.special_tokens_mask, + expected.n_sequences, + len(expected), + ) + assert copy.deepcopy(actual).ids == expected.ids + assert pickle.loads(pickle.dumps(actual)).offsets == expected.offsets + assert tokenizer.decode(actual.ids, skip_special_tokens=False) == reference.decode( + expected.ids, skip_special_tokens=False + ) + + +def test_huggingface_character_offsets_and_pretokenized_pairs_match_python() -> None: + reference: Final = ReferenceTokenizer.from_str(claude_json_str) + tokenizer: Final = HuggingFaceTokenizer.from_str(claude_json_str) + text: Final = "café 漢字 🙂" + actual: Final = tokenizer.encode(text) + expected: Final = reference.encode(text) + + assert actual.offsets == expected.offsets + assert actual.ids == expected.ids + assert ( + tokenizer.encode(["hello", "world"], ["again"], is_pretokenized=True).ids + == reference.encode(["hello", "world"], ["again"], is_pretokenized=True).ids + ) + + +def test_huggingface_batches_apply_padding_across_inputs() -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + reference.enable_padding(pad_id=0, pad_token="[UNK]") + tokenizer: Final = HuggingFaceTokenizer.from_str(reference.to_str()) + inputs: Final = ["Hello", ("Hello World", "World")] + expected: Final = reference.encode_batch(inputs) + actual: Final = tokenizer.encode_batch(inputs) + fast: Final = tokenizer.encode_batch_fast(inputs) + + assert [(item.ids, item.attention_mask, item.offsets) for item in actual] == [ + (item.ids, item.attention_mask, item.offsets) for item in expected + ] + assert [item.ids for item in fast] == [item.ids for item in expected] + assert tokenizer.decode_batch([item.ids for item in actual]) == reference.decode_batch( + [item.ids for item in expected] + ) + + +def test_caller_supplied_huggingface_tokenizer_preserves_public_encode_and_count() -> None: + tokenizer: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + custom: Final = {"type": "huggingface_tokenizer", "tokenizer": tokenizer} + expected: Final = tokenizer.encode("Hello World").ids + + assert litellm.encode(text="Hello World", custom_tokenizer=custom) == expected + assert litellm.token_counter(text="Hello World", custom_tokenizer=custom) == len(expected) + assert litellm.decode(tokens=expected, custom_tokenizer=custom) == "Hello World" + + +def test_caller_supplied_tiktoken_treats_special_spellings_as_text() -> None: + tokenizer: Final = tiktoken.get_encoding("cl100k_base") + custom: Final = {"type": "openai_tokenizer", "tokenizer": tokenizer} + text: Final = "<|endoftext|>" + + assert litellm.encode(text=text, custom_tokenizer=custom) == tokenizer.encode(text, disallowed_special=()) + + +def test_public_tokenizer_objects_survive_pickle_and_deepcopy(tmp_path: Path) -> None: + custom: Final = litellm.create_tokenizer(TOKENIZER_JSON) + tokenizer: Final = custom["tokenizer"] + path: Final = tmp_path / "tokenizer.json" + tokenizer.save(str(path)) + + assert copy.deepcopy(custom)["tokenizer"].encode("Hello World").ids == tokenizer.encode("Hello World").ids + assert ( + pickle.loads(pickle.dumps(custom))["tokenizer"].encode("Hello World").ids == tokenizer.encode("Hello World").ids + ) + assert HuggingFaceTokenizer.from_file(str(path)).encode("Hello World").ids == tokenizer.encode("Hello World").ids + assert copy.deepcopy(litellm.encoding).encode("hello") == litellm.encoding.encode("hello") + assert pickle.loads(pickle.dumps(litellm.encoding)).encode("hello") == litellm.encoding.encode("hello") + + +@pytest.mark.parametrize("offline", ("0", "1")) +def test_hub_loader_preserves_environment_auth_cache_and_offline(tmp_path: Path, offline: str) -> None: + script: Final = """ +import json +import sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +import httpx +import huggingface_hub +from huggingface_hub.errors import LocalEntryNotFoundError +import litellm +payload = sys.argv[2].encode() +offline = sys.argv[3] == "1" +observed = [] +def handle(request): + assert not offline, "offline loading issued a request" + if request.url.path.endswith("/tokenizer.json"): + observed.append(request.headers.get("authorization")) + if request.headers.get("authorization") != "Bearer audit-fixture-token": + return httpx.Response(401) + return httpx.Response(200, headers={"content-length": str(len(payload)), "etag": '"fixture"', "x-repo-commit": "a" * 40}, content=payload if request.method == "GET" else b"") +if not offline: + huggingface_hub.set_client_factory(lambda: httpx.Client(transport=httpx.MockTransport(handle))) +try: + tokenizer = litellm.create_pretrained_tokenizer("test-fixture/tokenizer")["tokenizer"] +except LocalEntryNotFoundError: + assert offline + assert observed == [] +else: + assert not offline + assert "Bearer audit-fixture-token" in observed + assert tokenizer.decode(tokenizer.encode("Hello World").ids) == "Hello World" + assert tuple(Path(sys.argv[4]).rglob("tokenizer.json")) +print("compatible") +""" + result: Final = subprocess.run( + [ + sys.executable, + "-I", + "-c", + script, + str(Path(litellm.__file__).parent.parent), + TOKENIZER_JSON, + offline, + str(tmp_path / "cache"), + ], + capture_output=True, + text=True, + timeout=30, + env={ + **os.environ, + "HF_HOME": str(tmp_path / "home"), + "HF_HUB_CACHE": str(tmp_path / "cache"), + "HF_ENDPOINT": "http://127.0.0.1:9", + "HF_TOKEN": "audit-fixture-token", + "HF_HUB_OFFLINE": offline, + "HF_HUB_DISABLE_IMPLICIT_TOKEN": "0", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "compatible" + + +@pytest.mark.parametrize("rust", (None, "0", "1")) +def test_tokenization_without_native_extension_stays_offline(tmp_path: Path, rust: str | None) -> None: + script: Final = """ +import importlib.abc +import sys +sys.path.insert(0, sys.argv[1]) +def reject_network(event, args): + if event == "socket.connect": + raise AssertionError("tokenizer attempted a network connection") +sys.addaudithook(reject_network) +class Block(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "litellm.rust_bridge._native": + raise ImportError("native extension is unavailable") +sys.meta_path.insert(0, Block()) +import litellm +from litellm.rust_bridge.tokenizer import get_encoding +import tiktoken +from tokenizers import Tokenizer +assert isinstance(litellm.encoding, tiktoken.Encoding) +for name in ("cl100k_base", "o200k_base", "o200k_harmony", "p50k_base", "p50k_edit"): + encoding = get_encoding(name) + text = "offline café 漢字 🙂" + " " * 64 + assert encoding.decode(encoding.encode(text)) == text +ids = litellm.encode(text="hello world") +assert litellm.decode(tokens=ids) == "hello world" +assert litellm.token_counter(model=None, text="hello world") == len(ids) +custom = litellm.create_tokenizer(sys.argv[2]) +assert isinstance(custom["tokenizer"], Tokenizer) +custom["tokenizer"].enable_padding(pad_id=0, pad_token="[UNK]") +assert litellm.decode(tokens=litellm.encode(text="Hello World", custom_tokenizer=custom), custom_tokenizer=custom) == "Hello World" +print("compatible") +""" + result: Final = subprocess.run( + [sys.executable, "-I", "-c", script, str(Path(litellm.__file__).parent.parent), TOKENIZER_JSON], + capture_output=True, + text=True, + timeout=30, + cwd=tmp_path, + env={ + **{key: value for key, value in os.environ.items() if key != "LITELLM_RUST"}, + **({"LITELLM_RUST": rust} if rust is not None else {}), + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + "TIKTOKEN_CACHE_DIR": str(tmp_path / "unused-tokenizer-cache"), + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "compatible" + assert not (tmp_path / "unused-tokenizer-cache").exists() + + +@pytest.mark.parametrize("is_pretokenized", (False, True)) +def test_huggingface_batch_sequence_containers_match_python(is_pretokenized: bool) -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + tokenizer: Final = HuggingFaceTokenizer.from_str(TOKENIZER_JSON) + inputs: Final = [["Hello", "World"], ("Hello", "World")] + actual: Final = tokenizer.encode_batch(inputs, is_pretokenized=is_pretokenized) + expected: Final = reference.encode_batch(inputs, is_pretokenized=is_pretokenized) + assert [(item.ids, item.type_ids, item.sequence_ids) for item in actual] == [ + (item.ids, item.type_ids, item.sequence_ids) for item in expected + ] + + +@pytest.mark.parametrize("name", ("cl100k_base", "o200k_base", "p50k_edit", "gpt2")) +def test_openai_encoding_exposes_the_tiktoken_vocabulary_surface(name: str) -> None: + reference: Final = tiktoken.get_encoding(name) + encoding: Final = OpenAIEncoding.from_tiktoken(name) + text: Final = "hello fanta" + + assert repr(encoding) == repr(reference) == f"" + assert (encoding.name, encoding.n_vocab, encoding.max_token_value) == ( + reference.name, + reference.n_vocab, + reference.max_token_value, + ) + assert encoding.token_byte_values() == reference.token_byte_values() + assert encoding.encode_single_token("hello") == reference.encode_single_token("hello") + assert encoding.encode_single_token(b"<|endoftext|>") == reference.eot_token + assert [encoding.is_special_token(token) for token in (0, reference.eot_token)] == [False, True] + assert encoding.decode_with_offsets(reference.encode(text)) == reference.decode_with_offsets(reference.encode(text)) + assert encoding.encode_to_numpy(text).tolist() == reference.encode_to_numpy(text).tolist() + stable, completions = encoding.encode_with_unstable(text) + expected_stable, expected_completions = reference.encode_with_unstable(text) + assert (stable, sorted(completions)) == (expected_stable, sorted(expected_completions)) + with pytest.raises(KeyError): + encoding.encode_single_token("<|not-a-token|>") + + +def test_huggingface_tokenizer_exposes_the_tokenizers_vocabulary_surface() -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + reference.enable_padding(pad_id=0, pad_token="[UNK]", length=4) + reference.enable_truncation(max_length=3, stride=1, strategy="only_first", direction="left") + tokenizer: Final = HuggingFaceTokenizer.from_str(reference.to_str()) + + assert tokenizer.token_to_id("Hello") == reference.token_to_id("Hello") == 1 + assert tokenizer.id_to_token(3) == reference.id_to_token(3) == "[BOS]" + assert tokenizer.id_to_token(99) is None + assert tokenizer.get_vocab() == reference.get_vocab() + assert tokenizer.get_vocab(with_added_tokens=False) == reference.get_vocab(with_added_tokens=False) + assert tokenizer.get_vocab_size() == reference.get_vocab_size() == 4 + assert tokenizer.get_vocab_size(with_added_tokens=False) == reference.get_vocab_size(with_added_tokens=False) + added: Final = tokenizer.get_added_tokens_decoder() + expected_added: Final = reference.get_added_tokens_decoder() + assert {token_id: str(token) for token_id, token in added.items()} == { + token_id: str(token) for token_id, token in expected_added.items() + } + assert added[3].special == expected_added[3].special + assert tokenizer.num_special_tokens_to_add(False) == reference.num_special_tokens_to_add(False) == 1 + assert tokenizer.num_special_tokens_to_add(True) == reference.num_special_tokens_to_add(True) == 0 + assert tokenizer.padding == reference.padding + assert tokenizer.truncation == reference.truncation + assert tokenizer.encode_special_tokens == reference.encode_special_tokens is False + assert HuggingFaceTokenizer.from_buffer(TOKENIZER_JSON.encode()).encode("Hello").ids == [3, 1] + assert HuggingFaceTokenizer.from_str(TOKENIZER_JSON).padding is None + assert HuggingFaceTokenizer.from_str(TOKENIZER_JSON).truncation is None + + +def test_huggingface_encoding_exposes_the_tokenizers_lookup_and_mutation_surface() -> None: + reference: Final = ReferenceTokenizer.from_str(claude_json_str) + tokenizer: Final = HuggingFaceTokenizer.from_str(claude_json_str) + text: Final = "hello wide world" + actual: Final = tokenizer.encode(text, "again") + expected: Final = reference.encode(text, "again") + + lookups: Final = ( + lambda encoding: [encoding.token_to_chars(index) for index in range(len(encoding))], + lambda encoding: [encoding.token_to_word(index) for index in range(len(encoding))], + lambda encoding: [encoding.token_to_sequence(index) for index in range(len(encoding))], + lambda encoding: [encoding.char_to_token(position) for position in range(len(text))], + lambda encoding: [encoding.char_to_word(position) for position in range(len(text))], + lambda encoding: [encoding.char_to_token(position, 1) for position in range(5)], + lambda encoding: [encoding.word_to_tokens(word) for word in range(3)], + lambda encoding: [encoding.word_to_chars(word) for word in range(3)], + lambda encoding: [encoding.word_to_tokens(0, 1), encoding.word_to_chars(0, 1)], + ) + for lookup in lookups: + assert lookup(actual) == lookup(expected) + assert repr(actual) == repr(expected) + + actual.truncate(4, stride=1, direction="left") + expected.truncate(4, stride=1, direction="left") + assert (actual.ids, [item.ids for item in actual.overflowing]) == ( + expected.ids, + [item.ids for item in expected.overflowing], + ) + actual.pad(6, direction="left", pad_id=7, pad_type_id=1, pad_token="") + expected.pad(6, direction="left", pad_id=7, pad_type_id=1, pad_token="") + assert (actual.ids, actual.attention_mask, actual.type_ids, actual.tokens) == ( + expected.ids, + expected.attention_mask, + expected.type_ids, + expected.tokens, + ) + actual.set_sequence_id(3) + expected.set_sequence_id(3) + assert actual.sequence_ids == expected.sequence_ids + merged: Final = type(actual).merge([actual, tokenizer.encode("more")]) + assert merged.ids == type(expected).merge([expected, reference.encode("more")]).ids + assert merged.offsets == type(expected).merge([expected, reference.encode("more")]).offsets + with pytest.raises(ValueError, match="direction"): + actual.pad(8, direction="sideways") diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 633dd1d9460..7167a67d80d 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,8 +1,12 @@ -import pytest - +import json +from typing import Final from unittest.mock import MagicMock, patch +import httpx +import pytest +import respx + import litellm from litellm.constants import ( ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, @@ -3771,6 +3775,70 @@ def test_multiple_compaction_blocks(): assert compaction_blocks[1]["content"] == "Second summary..." +@pytest.mark.parametrize("messages_api,gateway,native_endpoint", [ + (False, False, False), (True, False, False), (False, True, False), (True, True, False), (True, True, True), +]) +async def test_native_compaction_wire_roundtrip( + messages_api: bool, gateway: bool, native_endpoint: bool, + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True") + monkeypatch.setattr(litellm.anthropic_beta_headers_manager, "_BETA_HEADERS_CONFIG", None) + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", False) + block: Final = {"type": "compaction", "content": "Exact summary", "signature": "opaque-signature"} + operation: Final = {"type": "summarize", "instructions": "Keep identifiers"} + usage: Final = {"input_tokens": 0, "output_tokens": 0, + "iterations": [{"type": "compaction", "input_tokens": 103, "output_tokens": 165}]} + chat_wire: Final = gateway and not native_endpoint + base: Final = "https://gateway.test/v1" if gateway else "https://api.anthropic.com/v1" + route: Final = respx_mock.post(f"{base}/{'chat/completions' if chat_wire else 'messages'}") + + def respond(request: httpx.Request) -> httpx.Response: + payload: Final = json.loads(request.content) + assert len(request.headers.get_list("anthropic-beta")) == 1 + assert {value.strip() for value in request.headers["anthropic-beta"].split(",")} == { + "compact-2026-09-04", "interleaved-thinking-2025-05-14", + } + if "compaction" in payload: + assert payload["compaction"] == operation + else: + assert payload["messages"][0] == {"role": "assistant", "content": [block]} + body: Final = ( + {"id": "chatcmpl_compact", "object": "chat.completion", "created": 1, "model": "claude-sonnet-5", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "", + "provider_specific_fields": {"compaction_blocks": [block]}}}], + "usage": {"prompt_tokens": 103, "completion_tokens": 165, "total_tokens": 268}} + if chat_wire else + {"id": "msg_compact", "type": "message", "role": "assistant", "model": "claude-sonnet-5", + "content": [block], "stop_reason": "compaction", "usage": usage} + ) + return httpx.Response(200, json=body) + + route.mock(side_effect=respond) + call: Final = litellm.anthropic.messages.acreate if messages_api else litellm.acompletion + params: Final = dict( + model=f"{'openai/' if gateway else ''}anthropic/claude-sonnet-5", api_key="test", max_tokens=512, + api_base=base if gateway else "https://api.anthropic.com", + extra_headers={"Anthropic-Beta": f"interleaved-thinking-2025-05-14{',compact-2026-09-04' if gateway else ''}"}, + model_info={"supported_endpoints": ["/v1/messages"]} if native_endpoint else {}, + ) + response: Final = await call( + messages=[{"role": "user", "content": "Remember identifiers"}], compaction=operation, **params + ) + message: Final = response if messages_api else response.choices[0].message.model_dump() + blocks: Final = message["content"] if messages_api else message["provider_specific_fields"]["compaction_blocks"] + assert blocks == [block] + if messages_api: + assert response["stop_reason"] == "compaction" + if not chat_wire: + assert response["usage"] == usage + if not gateway: + replay: Final = {"role": "assistant", "content": blocks} if messages_api else message + await call(messages=[replay, {"role": "user", "content": "Continue"}], **params) + assert route.call_count == (1 if gateway else 2) + + def test_compaction_block_request_transformation(): """ Test that compaction blocks from provider_specific_fields are correctly diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index b9a82e3fc68..d03174bc2c6 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -60,6 +60,20 @@ def test_translate_openai_response_to_anthropic_empty_choices() -> None: assert result["usage"]["input_tokens"] == 10 +@pytest.mark.parametrize("text,count,expected_stop", [ + ("", 1, "compaction"), (None, 1, "compaction"), ("Answer", 1, "max_tokens"), + (" ", 1, "max_tokens"), ("", 2, "max_tokens"), ("", 0, "max_tokens"), +]) +def test_native_compaction_response_roundtrip(text: str | None, count: int, expected_stop: str) -> None: + block: Final = {"type": "compaction", "content": "Exact summary", "signature": "opaque-signature"} + message: Final = Message(content=text, provider_specific_fields={"compaction_blocks": [block] * count}) + response: Final = ModelResponse(choices=[Choices(message=message, finish_reason="length")], usage=Usage()) + result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response) + expected_text: Final = [{"type": "text", "text": text}] if text is not None and (text != "" or not count) else [] + assert result["content"] == [*([block] * count), *expected_text] + assert result["stop_reason"] == expected_stop + + def test_translate_chat_refusal_to_anthropic_response(): response = ModelResponse( id="chatcmpl-refusal", diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_reasoning_effort_normalization.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_reasoning_effort_normalization.py index af7befecc33..895b3b57f7b 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_reasoning_effort_normalization.py @@ -6,6 +6,8 @@ regression they guard is the one a caller sees: a tier the proxy advertises has leaves the adapter, in the shape the target expects. """ +from typing import Final + import pytest from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( @@ -36,6 +38,126 @@ def _reasoning_effort_sent(model: str, provider: str, reasoning_effort: object) return completion_kwargs.get("reasoning_effort") +def _reasoning_effort_sent_for_thinking( + model: str, + provider: str | None, + thinking: dict[str, object], + *, + tools: list[dict[str, object]] | None = None, + api_base: str | None = None, +) -> object: + extra_kwargs: Final = { + key: value for key, value in (("custom_llm_provider", provider), ("api_base", api_base)) if value is not None + } + completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=1024, + messages=MESSAGES, + model=model, + metadata=None, + stop_sequences=None, + stream=False, + system=None, + temperature=None, + thinking=thinking, + tool_choice=None, + tools=tools, + top_k=None, + top_p=None, + output_format=None, + extra_kwargs=extra_kwargs, + ) + return completion_kwargs.get("reasoning_effort") + + +SUMMARIZED_THINKING = {"type": "enabled", "budget_tokens": 4096, "summary": "auto"} +PLAIN_THINKING = {"type": "enabled", "budget_tokens": 4096} +MULTIPLY_TOOL = { + "name": "multiply", + "description": "Multiply two integers", + "input_schema": {"type": "object", "properties": {"a": {"type": "integer"}, "b": {"type": "integer"}}}, +} + + +class TestTheSummaryWrappingOnlyRidesTheResponsesBridge: + """Only the Responses API takes ``reasoning_effort`` as a dict. Databricks answered the wrapped + ``{"effort", "summary"}`` with ``field 'reasoning_effort' expects input with json type 'string' + but got 'object'``, so a target that stays on chat completions has to get the plain tier and a + target the bridge picks up has to keep the summary it can honor.""" + + @pytest.mark.parametrize( + "model, provider", + [ + ("databricks/databricks-qwen35-122b-a10b", "databricks"), + ("databricks-qwen35-122b-a10b", "databricks"), + ("fireworks_ai/kimi-k3", "fireworks_ai"), + ], + ) + def test_a_chat_target_gets_the_plain_tier(self, local_model_cost_map: None, model: str, provider: str) -> None: + assert _reasoning_effort_sent_for_thinking(model, provider, SUMMARIZED_THINKING) == "high" + + def test_auto_summary_stays_a_plain_tier_on_a_chat_target( + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true") + + sent = _reasoning_effort_sent_for_thinking("databricks/databricks-qwen35-122b-a10b", "databricks", PLAIN_THINKING) + + assert sent == "high" + + @pytest.mark.parametrize( + "model, provider", + [ + ("azure/responses/gpt-5-mini", "azure"), + ("gpt-5-mini", "openai"), + ("databricks/databricks-gpt-5-5", "databricks"), + ], + ) + def test_a_bridged_target_keeps_the_summary(self, local_model_cost_map: None, model: str, provider: str) -> None: + sent = _reasoning_effort_sent_for_thinking(model, provider, SUMMARIZED_THINKING) + + assert sent == {"effort": "high", "summary": "auto"} + + def test_auto_summary_still_reaches_a_bridged_target( + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true") + + sent = _reasoning_effort_sent_for_thinking("azure/responses/gpt-5-mini", "azure", PLAIN_THINKING) + + assert sent == {"effort": "high", "summary": "detailed"} + + @pytest.mark.parametrize( + "api_base, expected", + [ + ("https://foo.services.ai.azure.com/openai/v1", "high"), + ("https://foo.eastus.models.ai.azure.com", {"effort": "high", "summary": "auto"}), + ], + ) + def test_a_foundry_deployment_is_judged_by_its_api_base( + self, local_model_cost_map: None, api_base: str, expected: object + ) -> None: + """``completion()`` keeps a gpt-5.5 deployment with function tools on Foundry's chat route when + its ``api_base`` is a Foundry OpenAI host, and bridges it to Responses when the base makes it an + Azure OpenAI deployment. The adapter has to read the same ``api_base`` to land on the same call.""" + sent = _reasoning_effort_sent_for_thinking( + "azure_ai/gpt-5.5", "azure_ai", SUMMARIZED_THINKING, tools=[MULTIPLY_TOOL], api_base=api_base + ) + + assert sent == expected + + def test_a_provider_resolved_from_the_api_base_gets_the_plain_tier(self, local_model_cost_map: None) -> None: + sent = _reasoning_effort_sent_for_thinking( + "kimi-k3", None, SUMMARIZED_THINKING, api_base="https://api.together.xyz/v1" + ) + + assert sent == "high" + + def test_a_chained_gateway_keeps_the_dict_for_its_own_bridge(self, local_model_cost_map: None) -> None: + sent = _reasoning_effort_sent_for_thinking("litellm_proxy/gpt-5.4", "litellm_proxy", SUMMARIZED_THINKING) + + assert sent == {"effort": "high", "summary": "auto"} + + class TestTheNormalizedTierIsTheTierSent: """The bug in the caller's terms: a proxy advertising kimi-k3 ``max`` accepted the request and then put ``high`` on the wire. Every spelling of the entry has to survive the adapter, including diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_compact.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_compact.py index fc5d807bc23..d835db63d83 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_compact.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/context_management/test_compact.py @@ -16,6 +16,7 @@ import json from typing import Any, Dict, List from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import litellm @@ -1507,6 +1508,55 @@ async def test_summary_model_denied_when_team_member_scope_excludes_it(): assert result.applied_edits[0].get("error") == "summary_model_access_denied" +async def test_summary_model_denied_when_team_membership_read_hits_a_db_outage(): + """A member-level scope that cannot be read fails closed: the summary + model is not invoked while the membership row is unreachable.""" + messages = _simple_messages() + mock_call = AsyncMock(return_value=_make_mock_response("x")) + + auth = _fake_user_api_key_auth(key_models=["all-proxy-models"], team_id="team-outage") + auth.user_id = "user-outage" + + class _UnreachableMembershipPrisma: + class db: + class litellm_teammembership: + @staticmethod + async def find_unique(where: dict[str, dict[str, str]], include: dict[str, bool]) -> None: + raise httpx.ConnectError("All connection attempts failed") + + with ( + patch( + "litellm.llms.anthropic.experimental_pass_through.context_management.editors.compact._read_summary_model_setting", + return_value="claude-haiku-4-5", + ), + patch("litellm.token_counter", return_value=200_000), + patch( + "litellm.llms.anthropic.experimental_pass_through.context_management.editors.compact._call_summary_model", + mock_call, + ), + patch( + "litellm.proxy.auth.auth_checks.get_user_object", + AsyncMock(return_value=None), + ), + patch( + "litellm.proxy.auth.auth_checks.get_project_object", + AsyncMock(return_value=None), + ), + patch("litellm.proxy.proxy_server.prisma_client", _UnreachableMembershipPrisma()), + ): + result = await apply_compact_20260112( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec=_EDIT_SPEC_DEFAULT, + user_api_key_auth=auth, + ) + + mock_call.assert_not_awaited() + assert result.applied_edits[0].get("error") == "summary_model_access_denied" + + async def test_summary_model_denied_when_key_over_model_budget(): """A caller whose per-model budget for the summary model is exhausted cannot trigger the summary call via compaction.""" diff --git a/tests/test_litellm/llms/azure/realtime/__init__.py b/tests/test_litellm/llms/azure/realtime/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/tests/test_litellm/llms/azure/realtime/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/test_litellm/llms/azure/realtime/test_handler.py b/tests/test_litellm/llms/azure/realtime/test_handler.py new file mode 100644 index 00000000000..edf1b8b290f --- /dev/null +++ b/tests/test_litellm/llms/azure/realtime/test_handler.py @@ -0,0 +1,86 @@ +import json +from typing import cast +from unittest.mock import MagicMock, patch + +import pytest + + +class _RecordingClientWebSocket: + scope: dict[str, list[tuple[bytes, bytes]]] = {"headers": []} + + def __init__(self) -> None: + self.sent: list[str] = [] + self.closed: list[tuple[int, str | None]] = [] + + async def send_text(self, data: str) -> None: + self.sent.append(data) + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + self.closed.append((code, reason)) + + +@pytest.mark.asyncio +async def test_async_realtime_upstream_handshake_refusal_sends_error_event_then_policy_close(): + import websockets + + from litellm.llms.azure.realtime.handler import AzureOpenAIRealtime + from litellm.types.realtime import RealtimeErrorEvent + + handler = AzureOpenAIRealtime() + model = "gpt-realtime" + + dummy_websocket = _RecordingClientWebSocket() + dummy_logging_obj = MagicMock() + + refused = websockets.exceptions.InvalidStatus( + websockets.http11.Response(401, "Unauthorized", websockets.datastructures.Headers()) + ) + + with patch("websockets.connect", side_effect=refused): + await handler.async_realtime( # pyright: ignore[reportUnknownMemberType] # handler's websocket param is a Protocol here but the mock connect type is incomplete + model=model, + websocket=dummy_websocket, + logging_obj=dummy_logging_obj, + api_base="https://example.openai.azure.com", + api_key="bad-key", + api_version="2025-08-28", + query_params={"model": model}, + ) + + assert len(dummy_websocket.sent) == 1 + event = cast(RealtimeErrorEvent, json.loads(dummy_websocket.sent[0])) + assert event["type"] == "error" + assert event["error"]["type"] == "server_error" + assert "401" in event["error"]["message"] + assert dummy_websocket.closed and dummy_websocket.closed[0][0] == 1008 + + +@pytest.mark.asyncio +async def test_async_realtime_unexpected_error_sends_error_event_then_internal_close(): + from litellm.llms.azure.realtime.handler import AzureOpenAIRealtime + from litellm.types.realtime import RealtimeErrorEvent + + handler = AzureOpenAIRealtime() + model = "gpt-realtime" + + dummy_websocket = _RecordingClientWebSocket() + dummy_logging_obj = MagicMock() + + with patch("websockets.connect", side_effect=OSError("connection reset")): + await handler.async_realtime( # pyright: ignore[reportUnknownMemberType] # same as above + model=model, + websocket=dummy_websocket, + logging_obj=dummy_logging_obj, + api_base="https://example.openai.azure.com", + api_key="bad-key", + api_version="2025-08-28", + query_params={"model": model}, + ) + + assert len(dummy_websocket.sent) == 1 + event = cast(RealtimeErrorEvent, json.loads(dummy_websocket.sent[0])) + assert event["type"] == "error" + assert event["error"]["type"] == "server_error" + assert event["error"]["message"] == "Internal server error" + assert "connection reset" not in dummy_websocket.sent[0] + assert dummy_websocket.closed and dummy_websocket.closed[0] == (1011, "Internal server error") diff --git a/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py b/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py index 512e98b4151..fdd21c87732 100644 --- a/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py +++ b/tests/test_litellm/llms/azure_ai/image_generation/test_azure_ai_flux2_image_generation.py @@ -205,6 +205,36 @@ def test_flux2_flex_cost_accepts_lowercase_model_spelling(): assert cost == pytest.approx(5e-08 * 1536 * 1024 * 2) +def test_flux2_flex_cost_prefers_deployment_input_cost_per_pixel() -> None: + response: Final = ImageResponse(data=[ImageObject(b64_json="aW1n"), ImageObject(b64_json="aW1n")]) + + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model="FLUX.2-flex", + completion_response=response, + custom_llm_provider="azure_ai", + size="2048x1024", + call_type="image_generation", + model_info={"input_cost_per_pixel": 2e-07}, + ) + + assert cost == pytest.approx(2e-07 * 2048 * 1024 * 2) + + +def test_unlisted_azure_ai_model_bills_deployment_input_cost_per_pixel() -> None: + response: Final = ImageResponse(data=[ImageObject(b64_json="aW1n"), ImageObject(b64_json="aW1n")]) + + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model="unlisted-flux-deployment", + completion_response=response, + custom_llm_provider="azure_ai", + size="1024x1024", + call_type="image_generation", + model_info={"input_cost_per_pixel": 1e-07}, + ) + + assert cost == pytest.approx(1e-07 * 1024 * 1024 * 2) + + def test_flux2_response_preserves_mapped_dimensions(): config = AzureFoundryFluxImageGenerationConfig() params = config.map_openai_params( diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index db5da28c024..f92c610df24 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -5580,43 +5580,6 @@ def test_tool_config_cachepoint_not_placed_or_credited_for_model_without_prompt_ assert "litellm_gateway_injected_cache" not in bucket -def test_translate_response_format_json_schema_still_injects_tool(): - """ - response_format with an explicit json_schema should still use the - synthetic tool call approach (for models that don't support native - structured outputs). - """ - config = AmazonConverseConfig() - - response_format = { - "type": "json_schema", - "json_schema": { - "name": "FactResult", - "schema": { - "type": "object", - "properties": { - "facts": { - "type": "array", - "items": {"type": "string"}, - }, - }, - "required": ["facts"], - }, - }, - } - - optional_params: dict = {} - result = config._translate_response_format_param( - value=response_format, - model="anthropic.claude-3-haiku-20240307-v1:0", - optional_params=optional_params, - non_default_params={"response_format": response_format}, - is_thinking_enabled=False, - ) - - assert result["json_mode"] is True - assert "tools" in result - assert "tool_choice" in result def test_transform_response_finish_reason_stop_when_json_mode_filters_all_tools(): diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py index df042ce5902..117814a41ff 100644 --- a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py @@ -279,6 +279,16 @@ def test_context_window_suffix_stripped_for_cost_lookup(): ) +def test_legacy_mantle_route_prefix_stripped_for_cost_lookup(): + """The mantle/ route token is a routing prefix like openai/, so a bedrock/mantle/ + deployment must resolve the bare Bedrock model for cost lookup while still routing to Mantle.""" + from litellm.llms.bedrock.common_utils import get_bedrock_base_model, strip_bedrock_routing_prefix + + assert strip_bedrock_routing_prefix("mantle/anthropic.claude-sonnet-5") == "anthropic.claude-sonnet-5" + assert get_bedrock_base_model("bedrock/mantle/anthropic.claude-sonnet-5") == "anthropic.claude-sonnet-5" + assert BedrockModelInfo.get_bedrock_route("bedrock/mantle/anthropic.claude-sonnet-5") == "mantle" + + def test_output_config_effort_normalization_uses_model_info_ceiling(monkeypatch): import litellm.llms.bedrock.common_utils as mod @@ -926,3 +936,31 @@ def test_every_bedrock_config_get_error_class_keeps_provider_headers(config): def test_bedrock_get_error_class_audit_covers_every_surface(): assert len(_bedrock_configs_with_get_error_class()) >= 30 + + +def test_s3_static_key_pair_returns_the_pair_when_both_keys_are_set(): + from litellm.llms.bedrock.common_utils import s3_static_key_pair + + assert s3_static_key_pair( + { + "aws_access_key_id": "bedrock-key", + "aws_secret_access_key": "bedrock-secret", + "s3_access_key_id": "s3-key", + "s3_secret_access_key": "s3-secret", + } + ) == ("s3-key", "s3-secret") + + +@pytest.mark.parametrize( + "partial_s3_pair", + [ + {}, + {"s3_access_key_id": "s3-key"}, + {"s3_secret_access_key": "s3-secret"}, + {"s3_access_key_id": "", "s3_secret_access_key": ""}, + ], +) +def test_s3_static_key_pair_is_none_without_a_full_pair(partial_s3_pair): + from litellm.llms.bedrock.common_utils import s3_static_key_pair + + assert s3_static_key_pair({"aws_access_key_id": "bedrock-key", **partial_s3_pair}) is None diff --git a/tests/test_litellm/llms/bedrock/test_claude_platform_provider.py b/tests/test_litellm/llms/bedrock/test_claude_platform_provider.py index 40f78c84ca3..7b42d1f5a35 100644 --- a/tests/test_litellm/llms/bedrock/test_claude_platform_provider.py +++ b/tests/test_litellm/llms/bedrock/test_claude_platform_provider.py @@ -1,9 +1,62 @@ import json +from collections.abc import Callable, Mapping +from typing import Final from unittest.mock import MagicMock, patch import httpx import pytest from botocore.credentials import Credentials +from pydantic import BaseModel, ConfigDict, ValidationError + +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.types.router import GenericLiteLLMParams + +WORKSPACE_ALIASES: Final = ("workspace_id", "aws_workspace_id", "anthropic_workspace_id", "anthropic-workspace-id") + + +class ClaudePlatformMessagesBody(BaseModel): + """Messages API fields per https://docs.anthropic.com/en/api/messages (2026-09) minus context_management, which + the AWS endpoint rejects with 400""" + + model_config = ConfigDict(extra="forbid") + + model: str + messages: list[dict] + max_tokens: int + system: str | list[dict] | None = None + metadata: dict | None = None + stop_sequences: list[str] | None = None + stream: bool | None = None + temperature: float | None = None + top_k: int | None = None + top_p: float | None = None + tools: list[dict] | None = None + tool_choice: dict | None = None + thinking: dict | None = None + service_tier: str | None = None + mcp_servers: list[dict] | None = None + output_format: dict | None = None + container: str | dict | None = None + + +def _gateway_reject(url: str, message: str) -> httpx.Response: + return httpx.Response( + status_code=400, + json={"type": "error", "error": {"type": "invalid_request_error", "message": message}}, + request=httpx.Request("POST", url), + ) + + +def _fake_claude_platform_gateway(url: str, headers: Mapping[str, str], data: bytes | str | None) -> httpx.Response: + if "anthropic-workspace-id" not in headers: + return _gateway_reject(url, "missing anthropic-workspace-id header") + if "x-api-key" not in headers and not headers.get("Authorization", "").startswith("AWS4-HMAC-SHA256 "): + return _gateway_reject(url, "missing x-api-key or SigV4 Authorization") + try: + ClaudePlatformMessagesBody.model_validate_json(data or "{}") + except ValidationError as exc: + return _gateway_reject(url, "; ".join(f"{'.'.join(map(str, e['loc']))}: {e['msg']}" for e in exc.errors())) + return _anthropic_response(url) def _anthropic_response(url: str) -> httpx.Response: @@ -23,15 +76,39 @@ def _anthropic_response(url: str) -> httpx.Response: ) -def _capture_request(url: str, headers: dict, data: bytes | str | None) -> dict: +def _capture_request(url: str, headers: Mapping[str, str], data: bytes | str | None) -> dict: raw_body = data.decode("utf-8") if isinstance(data, bytes) else data or "{}" return { "path": httpx.URL(url).path, - "headers": headers, + "headers": httpx.Headers(dict(headers)), "body": json.loads(raw_body), } +GatewayResponder = Callable[[str, Mapping[str, str], bytes], httpx.Response] + + +def _gateway_transport(requests: list[dict], respond: GatewayResponder) -> httpx.MockTransport: + def handle(request: httpx.Request) -> httpx.Response: + requests.append(_capture_request(url=str(request.url), headers=request.headers, data=request.content)) + return respond(str(request.url), request.headers, request.content) + + return httpx.MockTransport(handle) + + +def _sync_gateway_client( + requests: list[dict], + respond: GatewayResponder = _fake_claude_platform_gateway, +) -> HTTPHandler: + return HTTPHandler(client=httpx.Client(transport=_gateway_transport(requests, respond))) + + +def _async_gateway_client(requests: list[dict]) -> AsyncHTTPHandler: + handler: Final = AsyncHTTPHandler() + handler.client = httpx.AsyncClient(transport=_gateway_transport(requests, _fake_claude_platform_gateway)) + return handler + + def test_claude_platform_builds_default_messages_url_from_region(): from litellm.llms.bedrock.claude_platform.transformation import ( BedrockClaudePlatformConfig, @@ -77,9 +154,7 @@ def test_claude_platform_uses_bedrock_subroute(): import litellm from litellm.llms.bedrock.common_utils import BedrockModelInfo - model, provider, _, _ = litellm.get_llm_provider( - model="bedrock/claude_platform/claude-sonnet-4-6" - ) + model, provider, _, _ = litellm.get_llm_provider(model="bedrock/claude_platform/claude-sonnet-4-6") assert provider == "bedrock" assert model == "claude_platform/claude-sonnet-4-6" @@ -177,9 +252,7 @@ def test_claude_platform_sigv4_signs_transformed_request_body(): assert signed_body == json.dumps(request_body).encode() assert headers["Authorization"] == "signed" mock_sign_request.assert_called_once() - assert ( - mock_sign_request.call_args.kwargs["service_name"] == "aws-external-anthropic" - ) + assert mock_sign_request.call_args.kwargs["service_name"] == "aws-external-anthropic" assert mock_sign_request.call_args.kwargs["request_data"] == request_body @@ -237,7 +310,7 @@ def test_bedrock_claude_platform_messages_config_round_trips_native_body(): model="claude_platform/claude-sonnet-4-6", messages=[{"role": "user", "content": "hello"}], anthropic_messages_optional_request_params={"max_tokens": 10}, - litellm_params={}, + litellm_params=GenericLiteLLMParams(), headers=headers, ) @@ -250,69 +323,6 @@ def test_bedrock_claude_platform_messages_config_round_trips_native_body(): } -def test_chat_completion_routes_bedrock_claude_platform_to_messages_api(): - import litellm - - requests = [] - - def mock_post(self, url, data=None, headers=None, **kwargs): - requests.append(_capture_request(url=url, headers=headers or {}, data=data)) - return _anthropic_response(url) - - with patch("litellm.llms.custom_httpx.http_handler.HTTPHandler.post", mock_post): - response = litellm.completion( - model="bedrock/claude_platform/claude-sonnet-4-6", - messages=[{"role": "user", "content": "hello"}], - max_tokens=10, - api_base="https://aws-external-anthropic.us-west-2.api.aws", - api_key="fake-platform-key", - workspace_id="wrkspc_test", - ) - - assert response.choices[0].message.content == "ok" - assert len(requests) == 1 - assert requests[0]["path"] == "/v1/messages" - assert requests[0]["headers"]["x-api-key"] == "fake-platform-key" - assert requests[0]["headers"]["anthropic-workspace-id"] == "wrkspc_test" - assert requests[0]["body"]["model"] == "claude-sonnet-4-6" - - -@pytest.mark.asyncio -async def test_anthropic_messages_routes_bedrock_claude_platform_to_messages_api(): - import litellm - - requests = [] - - async def mock_post(self, url, data=None, headers=None, **kwargs): - requests.append(_capture_request(url=url, headers=headers or {}, data=data)) - return _anthropic_response(url) - - try: - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - new=mock_post, - ): - response = await litellm.anthropic_messages( - model="bedrock/claude_platform/claude-sonnet-4-6", - messages=[{"role": "user", "content": "hello"}], - max_tokens=10, - api_base="https://aws-external-anthropic.us-west-2.api.aws", - api_key="fake-platform-key", - workspace_id="wrkspc_test", - ) - finally: - await litellm.close_litellm_async_clients() - - assert response["content"][0]["text"] == "ok" - assert len(requests) == 1 - assert requests[0]["path"] == "/v1/messages" - assert requests[0]["headers"]["x-api-key"] == "fake-platform-key" - assert requests[0]["headers"]["anthropic-workspace-id"] == "wrkspc_test" - assert requests[0]["body"]["messages"] == [{"role": "user", "content": "hello"}] - assert requests[0]["body"]["max_tokens"] == 10 - assert requests[0]["body"]["model"] == "claude-sonnet-4-6" - - @pytest.mark.asyncio async def test_anthropic_messages_bedrock_claude_platform_forwards_anthropic_beta_verbatim(): import litellm @@ -348,16 +358,380 @@ async def test_anthropic_messages_bedrock_claude_platform_forwards_anthropic_bet ] -def test_sigv4_no_duplicate_content_type_when_caller_sets_lowercase(): - """ - Regression: get_anthropic_headers() supplies "content-type" (lowercase). - _sign_request() used to prepend "Content-Type" (uppercase), leaving both - keys in the dict. botocore joins them into "application/json, application/json" - in the canonical string, while the wire request sends only one value → 401. +def test_claude_platform_strips_auth_params_from_request_body(): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) - Fix: prepend with lowercase "content-type" so **headers overwrites it when - the caller already set it. - """ + config = BedrockClaudePlatformConfig() + optional_params = { + "workspace_id": "wrkspc_test", + "aws_region_name": "us-west-2", + "max_tokens": 10, + } + + request_body = config.transform_request( + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert request_body == EXPECTED_CHAT_BODY + assert optional_params == {"workspace_id": "wrkspc_test", "aws_region_name": "us-west-2", "max_tokens": 10} + + +def test_claude_platform_messages_strips_auth_params_from_request_body(): + import litellm + from litellm.types.utils import LlmProviders + + config = litellm.ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude_platform/claude-sonnet-4-6", + provider=LlmProviders.BEDROCK, + ) + assert config is not None + + input_params = { + "workspace_id": "wrkspc_test", + "aws_region_name": "us-west-2", + "max_tokens": 10, + } + request_body = config.transform_anthropic_messages_request( + model="claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + anthropic_messages_optional_request_params=input_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert request_body == { + "model": "claude-sonnet-4-6", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 10, + } + assert input_params == {"workspace_id": "wrkspc_test", "aws_region_name": "us-west-2", "max_tokens": 10} + + +def test_claude_platform_strips_unsupported_context_management_param(caplog): + import logging + + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + config = BedrockClaudePlatformConfig() + optional_params = { + "workspace_id": "wrkspc_test", + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, + "max_tokens": 10, + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + request_body = config.transform_request( + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "context_management" not in request_body + assert request_body["max_tokens"] == 10 + assert "context_management" in optional_params + assert any( + "context_management" in record.message and record.levelno == logging.WARNING for record in caplog.records + ) + + +def test_claude_platform_messages_strips_unsupported_context_management_param(): + import litellm + from litellm.types.utils import LlmProviders + + config = litellm.ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude_platform/claude-sonnet-4-6", + provider=LlmProviders.BEDROCK, + ) + assert config is not None + + input_params = { + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, + "max_tokens": 10, + } + request_body = config.transform_anthropic_messages_request( + model="claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + anthropic_messages_optional_request_params=input_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert "context_management" not in request_body + assert request_body["max_tokens"] == 10 + assert "context_management" in input_params + + +@pytest.mark.parametrize("override_in", ["litellm_params", "optional_params"]) +def test_claude_platform_unsupported_override_allows_context_management(override_in): + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + config = BedrockClaudePlatformConfig() + override = {"claude_platform_unsupported_params": []} + context_management = {"edits": [{"type": "clear_tool_uses_20250919"}]} + + request_body = config.transform_request( + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params={ + "context_management": context_management, + "max_tokens": 10, + **(override if override_in == "optional_params" else {}), + }, + litellm_params=override if override_in == "litellm_params" else {}, + headers={}, + ) + + assert request_body == { + "model": "claude-sonnet-4-6", + "messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}], + "max_tokens": 10, + "context_management": context_management, + } + + +def test_claude_platform_unsupported_override_ignores_invalid_type(): + from litellm.llms.bedrock.claude_platform import common_utils + from litellm.llms.bedrock.claude_platform.transformation import ( + BedrockClaudePlatformConfig, + ) + + config = BedrockClaudePlatformConfig() + with patch.object(common_utils.verbose_logger, "warning") as mock_warning: + request_body = config.transform_request( + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params={ + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, + "max_tokens": 10, + }, + litellm_params={"claude_platform_unsupported_params": "not_a_list"}, + headers={}, + ) + + assert "context_management" not in request_body + assert request_body["max_tokens"] == 10 + warned = [call.args[0] for call in mock_warning.call_args_list] + assert any("claude_platform_unsupported_params" in message for message in warned) + + +def test_claude_platform_messages_does_not_advertise_beta_for_stripped_context_management(): + import litellm + from litellm.types.utils import LlmProviders + + config = litellm.ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude_platform/claude-sonnet-4-6", + provider=LlmProviders.BEDROCK, + ) + assert config is not None + + headers, _ = config.validate_anthropic_messages_environment( + api_key="fake-platform-key", + headers={}, + model="claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params={ + "max_tokens": 10, + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, + }, + litellm_params={"workspace_id": "wrkspc_test"}, + ) + + assert "context-management-2025-06-27" not in headers.get("anthropic-beta", "") + + +def test_claude_platform_messages_override_keeps_beta_for_context_management(): + import litellm + from litellm.types.utils import LlmProviders + + config = litellm.ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude_platform/claude-sonnet-4-6", + provider=LlmProviders.BEDROCK, + ) + assert config is not None + + headers, _ = config.validate_anthropic_messages_environment( + api_key="fake-platform-key", + headers={}, + model="claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + optional_params={ + "max_tokens": 10, + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, + }, + litellm_params={ + "workspace_id": "wrkspc_test", + "claude_platform_unsupported_params": [], + }, + ) + + assert "context-management-2025-06-27" in headers.get("anthropic-beta", "") + + +def test_claude_platform_messages_unsupported_override_allows_context_management(): + import litellm + from litellm.types.utils import LlmProviders + + config = litellm.ProviderConfigManager.get_provider_anthropic_messages_config( + model="claude_platform/claude-sonnet-4-6", + provider=LlmProviders.BEDROCK, + ) + assert config is not None + + request_body = config.transform_anthropic_messages_request( + model="claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + anthropic_messages_optional_request_params={ + "context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}, + "max_tokens": 10, + }, + litellm_params=GenericLiteLLMParams.model_validate({"claude_platform_unsupported_params": []}), + headers={}, + ) + + assert "context_management" in request_body + assert request_body["max_tokens"] == 10 + + +SIGV4_KWARGS: Final = { + "aws_region_name": "us-west-2", + "aws_access_key_id": "AKIATEST", + "aws_secret_access_key": "test-secret", + "aws_session_token": "test-token", +} +API_KEY_KWARGS: Final = {"api_key": "fake-platform-key", "aws_region_name": "us-west-2"} +EXPECTED_CHAT_BODY: Final = { + "model": "claude-sonnet-4-6", + "messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}], + "max_tokens": 10, +} +EXPECTED_NATIVE_BODY: Final = { + "model": "claude-sonnet-4-6", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 10, + "stream": False, +} + + +def _assert_gateway_accepted(request: dict, auth_kwargs: dict) -> None: + assert request["path"] == "/v1/messages" + assert request["headers"]["anthropic-workspace-id"] == "wrkspc_test" + if "api_key" in auth_kwargs: + assert request["headers"]["x-api-key"] == auth_kwargs["api_key"] + assert "Authorization" not in request["headers"] + else: + assert request["headers"]["Authorization"].startswith("AWS4-HMAC-SHA256 Credential=AKIATEST/") + assert "/us-west-2/aws-external-anthropic/aws4_request" in request["headers"]["Authorization"] + + +@pytest.mark.parametrize("auth_kwargs", [API_KEY_KWARGS, SIGV4_KWARGS], ids=["api_key", "sigv4"]) +@pytest.mark.parametrize("workspace_alias", WORKSPACE_ALIASES) +def test_chat_completion_claude_platform_sends_exact_body_through_strict_gateway(auth_kwargs, workspace_alias): + import litellm + + requests: list[dict] = [] + + response = litellm.completion( + model="bedrock/claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + max_tokens=10, + client=_sync_gateway_client(requests), + **{workspace_alias: "wrkspc_test"}, + **auth_kwargs, + ) + + assert response.choices[0].message.content == "ok" + assert len(requests) == 1, requests + assert requests[0]["body"] == EXPECTED_CHAT_BODY + _assert_gateway_accepted(requests[0], auth_kwargs) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("auth_kwargs", [API_KEY_KWARGS, SIGV4_KWARGS], ids=["api_key", "sigv4"]) +@pytest.mark.parametrize("workspace_alias", WORKSPACE_ALIASES) +async def test_anthropic_messages_claude_platform_sends_exact_body_through_strict_gateway(auth_kwargs, workspace_alias): + import litellm + + requests: list[dict] = [] + + response = await litellm.anthropic_messages( + model="bedrock/claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + max_tokens=10, + client=_async_gateway_client(requests), + **{workspace_alias: "wrkspc_test"}, + **auth_kwargs, + ) + + assert response["content"][0]["text"] == "ok" + assert len(requests) == 1, requests + assert requests[0]["body"] == EXPECTED_NATIVE_BODY + _assert_gateway_accepted(requests[0], auth_kwargs) + + +def test_chat_completion_claude_platform_drops_context_management_and_gateway_accepts(): + import litellm + + requests: list[dict] = [] + + litellm.completion( + model="bedrock/claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + max_tokens=10, + workspace_id="wrkspc_test", + context_management={"edits": [{"type": "clear_tool_uses_20250919"}]}, + client=_sync_gateway_client(requests), + **API_KEY_KWARGS, + ) + + assert requests[0]["body"] == EXPECTED_CHAT_BODY + + +def test_chat_completion_claude_platform_override_kwarg_is_honoured_and_not_sent(): + import litellm + + requests: list[dict] = [] + context_management = {"edits": [{"type": "clear_tool_uses_20250919"}]} + + litellm.completion( + model="bedrock/claude_platform/claude-sonnet-4-6", + messages=[{"role": "user", "content": "hello"}], + max_tokens=10, + workspace_id="wrkspc_test", + context_management=context_management, + claude_platform_unsupported_params=[], + client=_sync_gateway_client(requests, respond=lambda url, headers, data: _anthropic_response(url)), + **API_KEY_KWARGS, + ) + + assert requests[0]["body"] == {**EXPECTED_CHAT_BODY, "context_management": context_management} + + +def test_fake_claude_platform_gateway_rejects_leaked_internal_fields(): + leaked = json.dumps({**EXPECTED_NATIVE_BODY, "workspace_id": "wrkspc_test", "aws_region_name": "us-west-2"}) + response = _fake_claude_platform_gateway( + url="https://aws-external-anthropic.us-west-2.api.aws/v1/messages", + headers={"anthropic-workspace-id": "wrkspc_test", "x-api-key": "k"}, + data=leaked, + ) + assert response.status_code == 400 + assert response.json()["error"]["message"] == ( + "workspace_id: Extra inputs are not permitted; aws_region_name: Extra inputs are not permitted" + ) + + +def test_sigv4_no_duplicate_content_type_when_caller_sets_lowercase(): from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM llm = BaseAWSLLM() diff --git a/tests/test_litellm/llms/bedrock/test_mantle.py b/tests/test_litellm/llms/bedrock/test_mantle.py index 09be2118001..37cf49a85ec 100644 --- a/tests/test_litellm/llms/bedrock/test_mantle.py +++ b/tests/test_litellm/llms/bedrock/test_mantle.py @@ -447,6 +447,63 @@ async def test_mantle_anthropic_messages_sends_workspace_header_and_clean_body() assert "aws_bedrock_project_id" not in requests[0]["body"] +async def _send_anthropic_messages_with_betas(**request_params: object) -> dict: + import litellm + + requests = [] + + async def mock_post(self, url, data=None, headers=None, **kwargs): + requests.append(_capture_request(url=url, headers=headers or {}, data=data)) + return _anthropic_response(url) + + try: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new=mock_post, + ): + await litellm.anthropic_messages( + model="bedrock/mantle/anthropic.claude-mythos-preview", + messages=[{"role": "user", "content": "hello"}], + max_tokens=10, + aws_access_key_id="fake-key", + aws_secret_access_key="fake-secret", + aws_region_name="us-east-1", + **request_params, + ) + finally: + await litellm.close_litellm_async_clients() + + assert len(requests) == 1 + return requests[0] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_beta_headers_config") +async def test_mantle_anthropic_messages_sends_every_beta_in_the_header_not_the_body(): + sent = await _send_anthropic_messages_with_betas( + extra_headers={"anthropic-beta": "context-1m-2025-08-07,interleaved-thinking-2025-05-14"}, + context_management={"edits": [{"type": "clear_tool_uses_20250919"}]}, + ) + + assert ( + sent["headers"]["anthropic-beta"] + == "context-1m-2025-08-07,context-management-2025-06-27,interleaved-thinking-2025-05-14" + ) + assert sent["headers"]["anthropic-version"] == "2023-06-01" + assert sent["body"]["context_management"] == {"edits": [{"type": "clear_tool_uses_20250919"}]} + assert "anthropic_beta" not in sent["body"] + assert "anthropic_version" not in sent["body"] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_beta_headers_config") +async def test_mantle_anthropic_messages_drops_the_beta_header_when_mantle_rejects_every_value(): + sent = await _send_anthropic_messages_with_betas(extra_headers={"anthropic-beta": "code-execution-2025-08-25"}) + + assert "anthropic-beta" not in sent["headers"] + assert "anthropic_beta" not in sent["body"] + + def _usageless_anthropic_response(url: str) -> httpx.Response: return httpx.Response( status_code=200, diff --git a/tests/test_litellm/llms/custom_httpx/test_asgi_handler.py b/tests/test_litellm/llms/custom_httpx/test_asgi_handler.py new file mode 100644 index 00000000000..dbfa066a02d --- /dev/null +++ b/tests/test_litellm/llms/custom_httpx/test_asgi_handler.py @@ -0,0 +1,53 @@ +import asyncio +from collections.abc import Mapping +from typing import Final + +import pytest +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import JSONResponse, RedirectResponse +from starlette.routing import Route + +from litellm.llms.custom_httpx.asgi_handler import get_async_asgi_client + + +@pytest.mark.asyncio +async def test_cached_client_isolates_concurrent_apps_and_request_credentials() -> None: + ready: Final = (asyncio.Event(), asyncio.Event()) + + async def call(index: int) -> Mapping[str, object]: + async def endpoint(request: Request) -> JSONResponse: + ready[index].set() + await ready[1 - index].wait() + assert request.scope["root_path"] == f"/gateway-{index}" + assert request.client == (f"192.0.2.{index + 1}", 4321) + assert request.headers["authorization"] == f"Bearer key-{index}" + return JSONResponse({"app": index}, headers={"set-cookie": f"session=app-{index}; Path=/"}) + + app: Final = Starlette(routes=[Route("/child", endpoint, methods=["POST"])]) + with get_async_asgi_client(app, f"/gateway-{index}", (f"192.0.2.{index + 1}", 4321)) as client: + response: Final = await client.post( + f"https://proxy.test/gateway-{index}/child", headers={"authorization": f"Bearer key-{index}"}, + ) + assert response.status_code == 200 + assert not client.cookies + with get_async_asgi_client(app) as reused: + assert reused is client + return response.json() + + results: Final = await asyncio.wait_for(asyncio.gather(call(0), call(1)), timeout=5) + assert results == [{"app": 0}, {"app": 1}] + + +@pytest.mark.asyncio +async def test_internal_client_does_not_follow_redirects_or_environment_proxies(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("HTTPS_PROXY", "http://unreachable.invalid:8080") + + async def endpoint(request: Request) -> RedirectResponse: + return RedirectResponse("https://external.invalid/credentials") + + app: Final = Starlette(routes=[Route("/redirect", endpoint, methods=["POST"])]) + with get_async_asgi_client(app) as client: + response: Final = await client.post("https://proxy.test/redirect", headers={"authorization": "Bearer fixture"}) + assert response.status_code == 307 + assert not response.history diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index 33272a1a9e4..8358d15d30e 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -6,6 +6,8 @@ import pathlib import ssl import threading import weakref +from collections.abc import Callable, Mapping +from typing import Final from unittest.mock import MagicMock, patch import certifi @@ -23,6 +25,7 @@ from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, get_ssl_configuration, ) +from litellm.types.llms.custom_http import VerifyTypes @pytest.mark.asyncio @@ -1396,6 +1399,47 @@ async def test_finalizer_on_live_loop_disposes_foreign_loop_session_without_sche assert session.closed +class _RetryClientHandler(AsyncHTTPHandler): + def __init__(self, first: httpx.AsyncClient, retry: httpx.AsyncClient) -> None: + self._retry_client: Final = retry + super().__init__() + self.client = first + + def create_client( + self, + timeout: float | httpx.Timeout | None = None, + event_hooks: Mapping[str, list[Callable[..., object]]] | None = None, + ssl_verify: VerifyTypes | None = None, + shared_session: ClientSession | None = None, + ) -> httpx.AsyncClient: + return self._retry_client + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["post", "put", "patch", "delete"]) +async def test_connection_error_retry_forwards_content(method: str): + captured: list[bytes] = [] # mutable-ok: async closure capture buffer + + async def raise_connection_error(request: httpx.Request) -> httpx.Response: + raise httpx.RemoteProtocolError("connection dropped", request=request) + + async def capture_and_succeed(request: httpx.Request) -> httpx.Response: + captured.append(request.content) + return httpx.Response(200, request=request) + + first: Final = httpx.AsyncClient(transport=httpx.MockTransport(raise_connection_error)) + retry: Final = httpx.AsyncClient(transport=httpx.MockTransport(capture_and_succeed)) + async with first, retry: + handler: Final = _RetryClientHandler(first=first, retry=retry) + + body = b'{"post": ["run1"]}' + await getattr(handler, method)("https://api.example.com/runs/batch", content=body) + + assert captured == [body], "the retried request must carry the same content= body" + await handler.close() + + + @pytest.fixture def forward_proxy_server(): """Plain HTTP forward proxy that records the absolute URIs it is asked to fetch.""" diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index fa4b7439dd8..67a8d045036 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -4051,3 +4051,83 @@ async def test_responses_agentic_followup_does_not_repeat_request_params_from_pl assert followup_calls[0]["prompt_cache_key"] == "thread-1" assert followup_calls[0]["metadata"] == {"user": "u1"} assert followup_calls[0]["_agentic_loop_depth"] == 1 + + +@pytest.mark.asyncio +async def test_responses_agentic_followup_sends_the_plans_request_param_over_a_stale_kwargs_copy(monkeypatch): + from litellm.integrations.custom_logger import CustomLogger + from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch + + followup_calls: list[dict[str, object]] = [] + + async def fake_aresponses(**kwargs: object) -> str: + followup_calls.append(kwargs) + return "followup-response" + + monkeypatch.setattr(litellm, "aresponses", fake_aresponses) + + await BaseLLMHTTPHandler()._execute_responses_agentic_plan( + plan=AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + model="gpt-5", + messages=[{"role": "user", "content": "x"}], + optional_params={"prompt_cache_key": "from-plan-params"}, + kwargs={"prompt_cache_key": "stale-copy"}, + ), + ), + model="gpt-5", + response_api_optional_request_params={"prompt_cache_key": "from-request"}, + logging_obj=Mock(litellm_call_id="call-1"), + kwargs={}, + depth=0, + max_loops=3, + fingerprints=[], + fingerprint="fp", + callback=CustomLogger(), + ) + + assert followup_calls[0]["prompt_cache_key"] == "from-plan-params" + + +@pytest.mark.asyncio +async def test_chat_completion_agentic_followup_does_not_repeat_request_params_from_plan_kwargs(monkeypatch): + """A plan whose kwargs repeat a request param, or the explicitly passed model, must not crash the chat follow-up with a duplicate keyword""" + from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch + + followup_calls: list[dict[str, object]] = [] + + async def fake_acompletion(**kwargs: object) -> str: + followup_calls.append(kwargs) + return "followup-response" + + monkeypatch.setattr(litellm, "acompletion", fake_acompletion) + request_kwargs: Final = {"temperature": 0.2, "api_base": "https://a", "model": "gpt-5"} + plan: Final = AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + model="gpt-5", + messages=[{"role": "user", "content": "x"}], + optional_params={"temperature": 0.2}, + kwargs=dict(request_kwargs), + ), + ) + + response: Final = await BaseLLMHTTPHandler()._execute_chat_completion_agentic_plan( + plan=plan, + model="gpt-5", + messages=[{"role": "user", "content": "x"}], + optional_params={"temperature": 0.2}, + kwargs=dict(request_kwargs), + custom_llm_provider="openai", + depth=0, + max_loops=3, + fingerprints=[], + fingerprint="fp", + ) + + assert response == "followup-response" + assert len(followup_calls) == 1 + assert followup_calls[0]["temperature"] == 0.2 + assert followup_calls[0]["api_base"] == "https://a" + assert followup_calls[0]["model"] == "openai/gpt-5" diff --git a/litellm-rust/crates/cache-azure-blob/src/tests.rs b/tests/test_litellm/llms/databricks/chat/__init__.py similarity index 100% rename from litellm-rust/crates/cache-azure-blob/src/tests.rs rename to tests/test_litellm/llms/databricks/chat/__init__.py diff --git a/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py new file mode 100644 index 00000000000..a3391a2c585 --- /dev/null +++ b/tests/test_litellm/llms/databricks/chat/test_databricks_chat_transformation.py @@ -0,0 +1,79 @@ +import json +from typing import Final + +import httpx +import respx + +import litellm + + +def test_completion_merges_leading_system_and_developer_messages_for_chat_template_models( + respx_mock: respx.MockRouter, +): + upstream: Final = respx_mock.post("https://example.databricks.test/serving-endpoints/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "my-custom-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + response: Final = litellm.completion( + model="databricks/my-custom-model", + messages=[ + {"role": "system", "content": "You are terse."}, + {"role": "developer", "content": "Skills: none."}, + {"role": "user", "content": "Hello"}, + ], + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + ) + + assert upstream.call_count == 1 + request_body: Final = json.loads(upstream.calls[0].request.read()) + assert request_body["messages"] == [ + {"role": "system", "content": "You are terse.\n\nSkills: none."}, + {"role": "user", "content": "Hello"}, + ] + assert response.choices[0].message.content == "Answer" + + +def test_completion_merges_system_messages_when_one_has_empty_content(respx_mock: respx.MockRouter): + upstream: Final = respx_mock.post("https://example.databricks.test/serving-endpoints/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "my-custom-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + litellm.completion( + model="databricks/my-custom-model", + messages=[ + {"role": "system", "content": "You are terse."}, + {"role": "system", "content": ""}, + {"role": "user", "content": "Hello"}, + ], + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + ) + + request_body: Final = json.loads(upstream.calls[0].request.read()) + assert request_body["messages"] == [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hello"}, + ] diff --git a/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py b/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py index afac7b0bc1a..de0e547c0cd 100644 --- a/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py +++ b/tests/test_litellm/llms/databricks/test_databricks_cost_calculator.py @@ -153,14 +153,6 @@ def test_uncached_request_bills_every_prompt_token_at_the_input_rate(local_model assert completion_cost == pytest.approx(200 * info["output_cost_per_token"]) -def test_legacy_endpoint_names_still_resolve(local_model_cost_map: None) -> None: - info: Final = _model_info("databricks/databricks-mixtral-8x7b-instruct") - usage: Final = Usage(prompt_tokens=100, completion_tokens=100, total_tokens=200) - - prompt_cost, completion_cost = cost_per_token(model="databricks/mixtral-8x7b-instruct-v0.1", usage=usage) - - assert prompt_cost == pytest.approx(100 * info["input_cost_per_token"]) - assert completion_cost == pytest.approx(100 * info["output_cost_per_token"]) @pytest.mark.parametrize("model", NEW_MODELS) diff --git a/tests/test_litellm/llms/fal_ai/chat/test_fal_ai_chat_transformation.py b/tests/test_litellm/llms/fal_ai/chat/test_fal_ai_chat_transformation.py new file mode 100644 index 00000000000..b7a79f03f8d --- /dev/null +++ b/tests/test_litellm/llms/fal_ai/chat/test_fal_ai_chat_transformation.py @@ -0,0 +1,375 @@ +import httpx +import pytest + +import litellm +from litellm.llms.fal_ai.chat.transformation import FalAIChatConfig, FalAIError +from litellm.types.utils import LlmProviders, ModelResponse +from litellm.utils import ProviderConfigManager + +MODEL = "fal-ai/moondream3-preview/query" + + +@pytest.fixture(autouse=True) +def _use_local_model_cost_map(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +def _messages(*content): + return [ + { + "role": "user", + "content": [{"type": "text", "text": text} for text in content[:1]] + + [{"type": "image_url", "image_url": {"url": c}} for c in content[1:]], + } + ] + + +def test_provider_config_manager_resolves_fal_ai_chat_config(): + config = ProviderConfigManager.get_provider_chat_config(model=MODEL, provider=LlmProviders.FAL_AI) + assert isinstance(config, FalAIChatConfig) + + +def test_get_complete_url_targets_fal_endpoint(): + assert ( + FalAIChatConfig().get_complete_url( + api_base=None, api_key=None, model=MODEL, optional_params={}, litellm_params={} + ) + == "https://fal.run/fal-ai/moondream3-preview/query" + ) + + +def test_get_complete_url_strips_fal_ai_model_prefix(): + assert ( + FalAIChatConfig().get_complete_url( + api_base=None, api_key=None, model=f"fal_ai/{MODEL}", optional_params={}, litellm_params={} + ) + == "https://fal.run/fal-ai/moondream3-preview/query" + ) + + +def test_validate_environment_uses_fal_key_scheme(): + headers = FalAIChatConfig().validate_environment( + headers={}, model=MODEL, messages=[], optional_params={}, litellm_params={}, api_key="secret" + ) + assert headers["Authorization"] == "Key secret" + + +def test_transform_request_joins_text_parts_and_extracts_image_url(): + body = FalAIChatConfig().transform_request( + model=MODEL, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is"}, + {"type": "text", "text": "in this image?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/pic.png"}}, + ], + } + ], + optional_params={"temperature": 0.2, "top_p": 0.9, "reasoning": False}, + litellm_params={}, + headers={}, + ) + assert body == { + "prompt": "what is\nin this image?", + "image_url": "https://example.com/pic.png", + "temperature": 0.2, + "top_p": 0.9, + "reasoning": False, + } + + +def test_transform_request_passes_data_url_through(): + body = FalAIChatConfig().transform_request( + model=MODEL, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + ], + } + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert body["image_url"] == "data:image/png;base64,AAAA" + + +def test_transform_request_accepts_single_user_message(): + body = FalAIChatConfig().transform_request( + model=MODEL, + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "describe"}, {"type": "image_url", "image_url": "https://a"}], + } + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert body["prompt"] == "describe" + assert body["image_url"] == "https://a" + + +def test_transform_request_rejects_system_message(): + with pytest.raises(FalAIError, match="exactly one user message") as exc_info: + FalAIChatConfig().transform_request( + model=MODEL, + messages=[ + {"role": "system", "content": "be terse"}, + { + "role": "user", + "content": [{"type": "text", "text": "describe"}, {"type": "image_url", "image_url": "https://a"}], + }, + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert exc_info.value.status_code == 400 + + +def test_transform_request_rejects_multi_turn_history(): + with pytest.raises(FalAIError, match="exactly one user message") as exc_info: + FalAIChatConfig().transform_request( + model=MODEL, + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "first"}, {"type": "image_url", "image_url": "https://a"}], + }, + {"role": "assistant", "content": "an answer"}, + { + "role": "user", + "content": [{"type": "text", "text": "second"}, {"type": "image_url", "image_url": "https://b"}], + }, + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert exc_info.value.status_code == 400 + + +def test_transform_request_rejects_zero_images(): + with pytest.raises(FalAIError, match="exactly one image_url"): + FalAIChatConfig().transform_request( + model=MODEL, + messages=[{"role": "user", "content": [{"type": "text", "text": "describe"}]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + +def test_transform_request_rejects_two_images(): + with pytest.raises(FalAIError, match="exactly one image_url"): + FalAIChatConfig().transform_request( + model=MODEL, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "compare"}, + {"type": "image_url", "image_url": {"url": "https://a"}}, + {"type": "image_url", "image_url": {"url": "https://b"}}, + ], + } + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + + +def test_transform_request_rejects_missing_text(): + with pytest.raises(FalAIError, match="require text"): + FalAIChatConfig().transform_request( + model=MODEL, + messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://a"}}]}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + +def test_transform_request_rejects_streaming(): + with pytest.raises(FalAIError, match="streaming"): + FalAIChatConfig().transform_request( + model=MODEL, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe"}, + {"type": "image_url", "image_url": {"url": "https://a"}}, + ], + } + ], + optional_params={"stream": True}, + litellm_params={}, + headers={}, + ) + + +def test_completion_dispatch_rejects_streaming(): + with pytest.raises(litellm.BadRequestError): + litellm.completion( + model=MODEL, + custom_llm_provider="fal_ai", + stream=True, + messages=[{"role": "user", "content": "describe"}], + ) + + +@pytest.mark.parametrize( + "effort,expected", + [("none", False), ("minimal", False), ("low", True), ("medium", True), ("high", True)], +) +def test_map_openai_params_maps_reasoning_effort(effort, expected): + mapped = FalAIChatConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, optional_params={}, model=MODEL, drop_params=False + ) + assert mapped["reasoning"] is expected + + +def test_map_openai_params_drops_unknown_reasoning_effort_when_dropping(): + mapped = FalAIChatConfig().map_openai_params( + non_default_params={"reasoning_effort": "extreme"}, optional_params={}, model=MODEL, drop_params=True + ) + assert "reasoning" not in mapped + + +@pytest.mark.parametrize("effort", [{"level": "low"}, ["low"], 1]) +def test_map_openai_params_rejects_non_string_reasoning_effort(effort: object) -> None: + with pytest.raises(FalAIError) as exc_info: + FalAIChatConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, optional_params={}, model=MODEL, drop_params=False + ) + assert exc_info.value.status_code == 400 + + +@pytest.mark.parametrize("effort", [{"level": "low"}, ["low"], 1]) +def test_map_openai_params_drops_non_string_reasoning_effort_when_dropping(effort: object) -> None: + mapped = FalAIChatConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, optional_params={}, model=MODEL, drop_params=True + ) + assert "reasoning" not in mapped + + +def test_map_openai_params_maps_sampling_params(): + mapped = FalAIChatConfig().map_openai_params( + non_default_params={"temperature": 0.5, "top_p": 0.7, "max_tokens": 10}, + optional_params={}, + model=MODEL, + drop_params=False, + ) + assert mapped == {"temperature": 0.5, "top_p": 0.7} + + +def test_transform_response_maps_output_reasoning_usage_and_finish_reason(): + raw = httpx.Response( + 200, + json={ + "output": "a red circle", + "reasoning": "looked at shapes", + "finish_reason": "stop", + "usage_info": { + "input_tokens": 11, + "output_tokens": 4, + "prefill_time_ms": 1.0, + "decode_time_ms": 2.0, + "ttft_ms": 1.5, + }, + }, + ) + response = FalAIChatConfig().transform_response( + model=MODEL, + raw_response=raw, + model_response=ModelResponse(), + logging_obj=None, + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert response.choices[0].message.content == "a red circle" + assert response.choices[0].message.reasoning_content == "looked at shapes" + assert response.choices[0].finish_reason == "stop" + assert response.usage.prompt_tokens == 11 + assert response.usage.completion_tokens == 4 + assert response.usage.total_tokens == 15 + assert response.model == MODEL + + +def test_transform_response_omits_reasoning_when_null(): + raw = httpx.Response( + 200, + json={ + "output": "a red circle", + "reasoning": None, + "finish_reason": "stop", + "usage_info": {"input_tokens": 3, "output_tokens": 2}, + }, + ) + response = FalAIChatConfig().transform_response( + model=MODEL, + raw_response=raw, + model_response=ModelResponse(), + logging_obj=None, + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert response.choices[0].message.content == "a red circle" + assert getattr(response.choices[0].message, "reasoning_content", None) is None + assert response.usage.total_tokens == 5 + + +def test_transform_response_rejects_body_missing_output(): + raw = httpx.Response( + 200, + json={"reasoning": "looked", "usage_info": {"input_tokens": 3, "output_tokens": 2}}, + ) + with pytest.raises(FalAIError) as exc_info: + FalAIChatConfig().transform_response( + model=MODEL, + raw_response=raw, + model_response=ModelResponse(), + logging_obj=None, + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert exc_info.value.status_code == 422 + + +def test_transform_response_rejects_body_missing_usage_info(): + raw = httpx.Response(200, json={"output": "a red circle"}) + with pytest.raises(FalAIError) as exc_info: + FalAIChatConfig().transform_response( + model=MODEL, + raw_response=raw, + model_response=ModelResponse(), + logging_obj=None, + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert exc_info.value.status_code == 422 diff --git a/tests/test_litellm/llms/fal_ai/image_edit/test_fal_ai_flux_lora_depth_transformation.py b/tests/test_litellm/llms/fal_ai/image_edit/test_fal_ai_flux_lora_depth_transformation.py new file mode 100644 index 00000000000..d99701db9e8 --- /dev/null +++ b/tests/test_litellm/llms/fal_ai/image_edit/test_fal_ai_flux_lora_depth_transformation.py @@ -0,0 +1,116 @@ +import base64 +import io + +import pytest + +import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils +from litellm.llms.fal_ai.image_edit import ( + FalAIFluxLoraDepthEditConfig, + FalAIImageEditConfig, + get_fal_ai_image_edit_config, +) +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import ImageObject, ImageResponse, LlmProviders +from litellm.utils import ProviderConfigManager + +PNG_BYTES = b"\x89PNG\r\n\x1a\n" + b"\x00" * 16 +MODEL = "fal-ai/flux-lora-depth" + + +@pytest.fixture(autouse=True) +def _use_local_model_cost_map(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +@pytest.mark.parametrize("model", ["fal-ai/flux-lora-depth", "flux-lora-depth", "fal_ai/fal-ai/flux-lora-depth"]) +def test_dispatch_selects_flux_lora_depth_config(model): + assert isinstance(get_fal_ai_image_edit_config(model), FalAIFluxLoraDepthEditConfig) + + +def test_dispatch_keeps_gpt_image_config_for_openai_edit_models(): + config = get_fal_ai_image_edit_config("openai/gpt-image-2.5/flare/edit") + assert type(config) is FalAIImageEditConfig + + +def test_provider_config_manager_resolves_flux_lora_depth(): + config = ProviderConfigManager.get_provider_image_edit_config(model=MODEL, provider=LlmProviders.FAL_AI) + assert isinstance(config, FalAIFluxLoraDepthEditConfig) + + +@pytest.mark.parametrize("model", ["fal-ai/flux-lora-depth", "flux-lora-depth"]) +def test_get_complete_url_targets_endpoint_without_edit_suffix(model): + url = FalAIFluxLoraDepthEditConfig().get_complete_url(model=model, api_base=None, litellm_params={}) + assert url == "https://fal.run/fal-ai/flux-lora-depth" + + +def test_get_supported_openai_params_excludes_quality_mask_background(): + params = FalAIFluxLoraDepthEditConfig().get_supported_openai_params(model=MODEL) + assert "quality" not in params + assert "mask" not in params + assert "background" not in params + + +def test_map_openai_params_translates_n_and_size(): + mapped = FalAIFluxLoraDepthEditConfig().map_openai_params( + image_edit_optional_params=ImageEditOptionalRequestParams(n=2, size="1024x1536", quality="high"), + model=MODEL, + drop_params=False, + ) + assert mapped == {"num_images": 2, "image_size": {"width": 1024, "height": 1536}} + + +def test_transform_request_sends_single_image_url_as_data_url(): + body, files = FalAIFluxLoraDepthEditConfig().transform_image_edit_request( + model=MODEL, + prompt="follow the depth map", + image=io.BytesIO(PNG_BYTES), + image_edit_optional_request_params={"num_images": 1}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert files == () + assert body["prompt"] == "follow the depth map" + assert body["image_url"] == "data:image/png;base64," + base64.b64encode(PNG_BYTES).decode() + assert "image_urls" not in body + assert body["num_images"] == 1 + + +def test_transform_request_passes_remote_url_through_untouched(): + body, _ = FalAIFluxLoraDepthEditConfig().transform_image_edit_request( + model=MODEL, + prompt="follow the depth map", + image="https://example.com/depth.png", + image_edit_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert body["image_url"] == "https://example.com/depth.png" + + +def test_transform_request_rejects_two_images(): + with pytest.raises(ValueError, match="exactly one control image"): + FalAIFluxLoraDepthEditConfig().transform_image_edit_request( + model=MODEL, + prompt="follow the depth map", + image=["https://example.com/a.png", "https://example.com/b.png"], + image_edit_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + +def test_image_edit_cost_uses_flat_output_cost_per_image(): + cost = CostCalculatorUtils.route_image_generation_cost_calculator( + model=MODEL, + completion_response=ImageResponse(data=[ImageObject(url="https://example.com/out.png")]), + custom_llm_provider="fal_ai", + optional_params={}, + call_type="aimage_edit", + ) + assert cost == litellm.model_cost[f"fal_ai/{MODEL}"]["output_cost_per_image"] > 0 diff --git a/tests/test_litellm/llms/fal_ai/test_cost_calculator.py b/tests/test_litellm/llms/fal_ai/test_cost_calculator.py index 6fb34d9f88e..3c6e6aea090 100644 --- a/tests/test_litellm/llms/fal_ai/test_cost_calculator.py +++ b/tests/test_litellm/llms/fal_ai/test_cost_calculator.py @@ -1,8 +1,10 @@ +from typing import Final + import pytest import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils -from litellm.llms.fal_ai.cost_calculator import cost_calculator +from litellm.llms.fal_ai.cost_calculator import cost_calculator, fal_ai_passthrough_cost from litellm.types.utils import ImageObject, ImageResponse @@ -78,14 +80,27 @@ def test_gpt_image_response_dimensions_override_request_size(): assert cost == expected -def test_gpt_image_response_dimensions_fall_back_to_request_size_when_unpriced(): +def test_gpt_image_response_dimensions_use_nearest_keyed_row_when_unpriced(): model = "fal_ai/openai/gpt-image-2.5/flare/text-to-image" cost = cost_calculator( model=model, image_response=_image_response_with_dimensions(((777, 888),)), optional_params={"quality": "low", "image_size": {"width": 1024, "height": 1536}}, ) - expected = litellm.model_cost[f"fal_ai/low/1024-x-1536/{model.removeprefix('fal_ai/')}"]["output_cost_per_image"] + expected = litellm.model_cost[f"fal_ai/low/1024-x-768/{model.removeprefix('fal_ai/')}"]["output_cost_per_image"] + assert cost == expected + + +def test_gpt_image_25_noncanonical_response_uses_nearest_keyed_row(): + model: Final = "fal_ai/openai/gpt-image-2.5/flare/text-to-image" + cost: Final = cost_calculator( + model=model, + image_response=_image_response_with_dimensions(((1536, 1024),)), + optional_params={"quality": "low", "image_size": {"width": 1536, "height": 1024}}, + ) + expected: Final = litellm.model_cost[ + "fal_ai/low/1024-x-1536/openai/gpt-image-2.5/flare/text-to-image" + ]["output_cost_per_image"] assert cost == expected @@ -160,3 +175,70 @@ def test_image_edit_call_type_routes_to_fal_keyed_pricing(): call_type="aimage_edit", ) assert cost == litellm.model_cost[f"fal_ai/medium/1024-x-1024/{model}"]["output_cost_per_image"] > 0 + + +def test_passthrough_trellis_charges_flat_rate(): + assert ( + fal_ai_passthrough_cost("fal-ai/trellis", {}) + == litellm.model_cost["fal_ai/fal-ai/trellis"]["output_cost_per_image"] + > 0 + ) + + +@pytest.mark.parametrize("resolution", [512, 1024, 1536]) +def test_passthrough_trellis_2_resolution_picks_keyed_tier(resolution): + assert ( + fal_ai_passthrough_cost("fal-ai/trellis-2", {"resolution": resolution}) + == litellm.model_cost["fal_ai/fal-ai/trellis-2"][f"output_cost_per_image_{resolution}"] + > 0 + ) + + +def test_passthrough_trellis_2_without_resolution_falls_back_to_default_rate(): + assert ( + fal_ai_passthrough_cost("fal-ai/trellis-2", {"image_url": "https://a"}) + == litellm.model_cost["fal_ai/fal-ai/trellis-2"]["output_cost_per_image"] + > 0 + ) + + +def test_passthrough_unknown_model_returns_none(): + assert fal_ai_passthrough_cost("fal-ai/no-such-model", {"resolution": 512}) is None + + +def test_passthrough_string_resolution_is_priced_like_the_integer(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setitem( + litellm.model_cost, + "fal_ai/fal-ai/keyed-model", + { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.3, + "output_cost_per_image_512": 0.25, + "output_cost_per_image_1536": 0.35, + }, + ) + assert fal_ai_passthrough_cost("fal-ai/keyed-model", {"resolution": "512"}) == 0.25 + assert fal_ai_passthrough_cost("fal-ai/keyed-model", {"resolution": 512}) == 0.25 + assert fal_ai_passthrough_cost("fal-ai/keyed-model", {"resolution": "1536"}) == 0.35 + assert fal_ai_passthrough_cost("fal-ai/keyed-model", {"resolution": True}) == 0.3 + assert fal_ai_passthrough_cost("fal-ai/keyed-model", {"resolution": 512.0}) == 0.3 + + +def test_passthrough_cost_is_none_only_when_no_price_applies_to_the_request(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setitem( + litellm.model_cost, + "fal_ai/fal-ai/priceless-model", + {"litellm_provider": "fal_ai", "mode": "image_generation"}, + ) + monkeypatch.setitem( + litellm.model_cost, + "fal_ai/fal-ai/keyed-only-model", + {"litellm_provider": "fal_ai", "mode": "image_generation", "output_cost_per_image_512": 0.02}, + ) + assert fal_ai_passthrough_cost("fal-ai/priceless-model", {}) is None + assert fal_ai_passthrough_cost("fal-ai/priceless-model", {"resolution": 512}) is None + assert fal_ai_passthrough_cost("fal-ai/no-such-model", {}) is None + assert fal_ai_passthrough_cost("fal-ai/keyed-only-model", {}) is None + assert fal_ai_passthrough_cost("fal-ai/keyed-only-model", {"resolution": 1024}) is None + assert fal_ai_passthrough_cost("fal-ai/keyed-only-model", {"resolution": "512"}) == 0.02 diff --git a/tests/test_litellm/llms/fal_ai/videos/test_fal_ai_video_transformation.py b/tests/test_litellm/llms/fal_ai/videos/test_fal_ai_video_transformation.py index 86ecbf6701b..d4ad22ca5a0 100644 --- a/tests/test_litellm/llms/fal_ai/videos/test_fal_ai_video_transformation.py +++ b/tests/test_litellm/llms/fal_ai/videos/test_fal_ai_video_transformation.py @@ -1,3 +1,4 @@ +import sys from typing import Final from unittest.mock import AsyncMock, Mock @@ -86,6 +87,17 @@ class TestFalAIVideoTransformation: assert mapped["reference_image_urls"] == [url] assert "image_url" not in mapped + def test_map_openai_params_h3_omits_auto_duration(self): + assert self.config.map_openai_params({"seconds": "auto"}, H3_TEXT_MODEL, False) == {} + assert self.config.map_openai_params({"seconds": "auto"}, MODEL, False) == {"duration": "auto"} + + def test_map_openai_params_h3_size_beyond_tiers_uses_top_resolution(self): + side = str(sys.maxsize + 1) + assert self.config.map_openai_params({"size": f"{side}x{side}"}, H3_TEXT_MODEL, False) == { + "resolution": "4K", + "aspect_ratio": "1:1", + } + def test_transform_video_create_request(self): body, files, url = self.config.transform_video_create_request( model=MODEL, @@ -377,6 +389,117 @@ class TestFalAIVideoTransformation: assert "input.reference_image_urls: Failed to download the file" in video.error["message"] client.get.assert_awaited_once_with(url=result_url, headers=auth_headers) + def test_status_completed_result_probe_reuses_status_client_and_extra_headers(self): + status_url = "https://queue.fal.run/minimax/h3/requests/abc/status" + status_headers: Final = { + "Authorization": "Key synthetic-fal-key", + "Content-Type": "application/json", + "X-Routing": "canary-7", + } + response: Final = httpx.Response( + 200, + json={"request_id": "abc", "status": "COMPLETED"}, + request=httpx.Request("GET", status_url, headers=status_headers), + ) + result_url: Final = status_url.removesuffix("/status") + status_client: Final = Mock() + status_client.get.return_value = httpx.Response( + 403, text="missing X-Routing", request=httpx.Request("GET", result_url) + ) + factory_client: Final = Mock() + config = FalAIVideoConfig(sync_client_factory=lambda: factory_client) + + video = config.transform_video_status_retrieve_response( + raw_response=response, + logging_obj=self.logging_obj, + custom_llm_provider="fal_ai", + client=status_client, + ) + + assert video.status == "failed" + assert video.error == {"code": "fal_error", "message": "missing X-Routing"} + factory_client.get.assert_not_called() + status_client.get.assert_called_once() + assert status_client.get.call_args.kwargs["url"] == result_url + assert status_client.get.call_args.kwargs["headers"].items() >= status_headers.items() + + def test_status_completed_result_probe_transport_error_keeps_completed(self): + status_url = "https://queue.fal.run/minimax/h3/requests/abc/status" + response: Final = httpx.Response( + 200, + json={"request_id": "abc", "status": "COMPLETED"}, + request=httpx.Request("GET", status_url), + ) + client: Final = Mock() + client.get.side_effect = httpx.ReadError("connection reset by fal.ai") + + video = FalAIVideoConfig().transform_video_status_retrieve_response( + raw_response=response, + logging_obj=self.logging_obj, + custom_llm_provider="fal_ai", + client=client, + ) + + assert video.status == "completed" + assert video.error is None + + @pytest.mark.asyncio + async def test_async_status_completed_result_probe_reuses_status_client_and_extra_headers(self): + status_url = "https://queue.fal.run/minimax/h3/requests/abc/status" + status_headers: Final = { + "Authorization": "Key synthetic-fal-key", + "Content-Type": "application/json", + "X-Routing": "canary-7", + } + response: Final = httpx.Response( + 200, + json={"request_id": "abc", "status": "COMPLETED"}, + request=httpx.Request("GET", status_url, headers=status_headers), + ) + result_url: Final = status_url.removesuffix("/status") + status_client: Final = Mock() + status_client.get = AsyncMock( + return_value=httpx.Response(403, text="missing X-Routing", request=httpx.Request("GET", result_url)) + ) + factory_client: Final = Mock() + factory_client.get = AsyncMock() + config = FalAIVideoConfig(async_client_factory=lambda: factory_client) + + video = await config.async_transform_video_status_retrieve_response( + raw_response=response, + logging_obj=self.logging_obj, + custom_llm_provider="fal_ai", + client=status_client, + ) + + assert video.status == "failed" + assert video.error == {"code": "fal_error", "message": "missing X-Routing"} + factory_client.get.assert_not_awaited() + status_client.get.assert_awaited_once() + assert status_client.get.await_args.kwargs["url"] == result_url + assert status_client.get.await_args.kwargs["headers"].items() >= status_headers.items() + + @pytest.mark.asyncio + async def test_async_status_completed_result_probe_transport_error_keeps_completed(self): + status_url = "https://queue.fal.run/minimax/h3/requests/abc/status" + response: Final = httpx.Response( + 200, + json={"request_id": "abc", "status": "COMPLETED"}, + request=httpx.Request("GET", status_url), + ) + client: Final = Mock() + client.get = AsyncMock(side_effect=httpx.ConnectError("tls handshake failed")) + + video = await FalAIVideoConfig().async_transform_video_status_retrieve_response( + raw_response=response, + logging_obj=self.logging_obj, + custom_llm_provider="fal_ai", + client=client, + ) + + assert video.status == "completed" + assert video.error is None + def test_status_in_progress_does_not_fetch_result(self): status_url = "https://queue.fal.run/minimax/h3/requests/abc/status" response = httpx.Response( @@ -552,17 +675,23 @@ class TestFalAIVideoTransformation: row = litellm.model_cost[f"fal_ai/{H3_TEXT_MODEL}"] model_info = litellm.get_model_info(model=H3_TEXT_MODEL, custom_llm_provider="fal_ai") - assert video_generation_cost( - model=H3_TEXT_MODEL, - duration_seconds=5, - custom_llm_provider="fal_ai", - model_info=model_info, - video_resolution="2K", - ) == 5 * row["output_cost_per_second_2k"] - assert video_generation_cost( - model=H3_TEXT_MODEL, - duration_seconds=5, - custom_llm_provider="fal_ai", - model_info=model_info, - video_resolution="768p", - ) == 5 * row["output_cost_per_second_768p"] + assert ( + video_generation_cost( + model=H3_TEXT_MODEL, + duration_seconds=5, + custom_llm_provider="fal_ai", + model_info=model_info, + video_resolution="2K", + ) + == 5 * row["output_cost_per_second_2k"] + ) + assert ( + video_generation_cost( + model=H3_TEXT_MODEL, + duration_seconds=5, + custom_llm_provider="fal_ai", + model_info=model_info, + video_resolution="768p", + ) + == 5 * row["output_cost_per_second_768p"] + ) diff --git a/tests/test_litellm/llms/gemini/test_cost_calculator.py b/tests/test_litellm/llms/gemini/test_cost_calculator.py index 5eed11dff03..b2633c6091b 100644 --- a/tests/test_litellm/llms/gemini/test_cost_calculator.py +++ b/tests/test_litellm/llms/gemini/test_cost_calculator.py @@ -200,172 +200,12 @@ def test_maps_no_usage_details(): assert cost_per_google_maps_grounding_request(usage=usage, model_info=model_info) == 0.0 -def test_gemini_image_edit_cost_prefers_token_usage_metadata(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - model = "gemini/gemini-3-pro-image-preview" - model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") - - input_text_tokens = 20 - input_image_tokens = 1120 - output_image_tokens = 1120 - prompt_tokens = input_text_tokens + input_image_tokens - image_response = ImageResponse( - data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")], - usage=ImageUsage( - input_tokens=prompt_tokens, - input_tokens_details=ImageUsageInputTokensDetails( - text_tokens=input_text_tokens, - image_tokens=input_image_tokens, - ), - output_tokens=output_image_tokens, - total_tokens=prompt_tokens + output_image_tokens, - ), - ) - - cost = gemini_image_edit_cost_calculator( - model=model, - image_response=image_response, - ) - - expected_cost = ( - prompt_tokens * model_info["input_cost_per_token"] - + output_image_tokens * model_info["output_cost_per_image_token"] - ) - flat_image_cost = ( - len(image_response.data or []) * model_info["output_cost_per_image"] - ) - assert round(cost, 10) == round(expected_cost, 10) - assert cost != flat_image_cost -def test_gemini_image_edit_cost_uses_output_token_details(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - model = "gemini/gemini-3-pro-image-preview" - model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") - - input_text_tokens = 20 - output_text_tokens = 213 - output_image_tokens = 1120 - output_tokens = output_text_tokens + output_image_tokens - image_response = ImageResponse( - data=[ImageObject(b64_json="img1")], - usage=ImageUsage( - input_tokens=input_text_tokens, - input_tokens_details=ImageUsageInputTokensDetails( - text_tokens=input_text_tokens, - image_tokens=0, - ), - output_tokens=output_tokens, - total_tokens=input_text_tokens + output_tokens, - prompt_tokens=input_text_tokens, - completion_tokens=output_tokens, - prompt_tokens_details={ - "text_tokens": input_text_tokens, - "image_tokens": 0, - }, - completion_tokens_details={ - "text_tokens": output_text_tokens, - "image_tokens": output_image_tokens, - }, - output_tokens_details={ - "text_tokens": output_text_tokens, - "image_tokens": output_image_tokens, - }, - ), - ) - - cost = gemini_image_edit_cost_calculator( - model=model, - image_response=image_response, - ) - - expected_cost = ( - input_text_tokens * model_info["input_cost_per_token"] - + output_text_tokens * model_info["output_cost_per_token"] - + output_image_tokens * model_info["output_cost_per_image_token"] - ) - all_output_as_image_cost = ( - input_text_tokens * model_info["input_cost_per_token"] - + (output_text_tokens + output_image_tokens) - * model_info["output_cost_per_image_token"] - ) - assert round(cost, 10) == round(expected_cost, 10) - assert cost != all_output_as_image_cost -def test_gemini_image_generation_cost_uses_output_token_details(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - model = "gemini/gemini-3-pro-image-preview" - model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") - - input_text_tokens = 20 - output_text_tokens = 213 - output_image_tokens = 1120 - output_tokens = output_text_tokens + output_image_tokens - image_response = ImageResponse( - data=[ImageObject(b64_json="img1")], - usage=ImageUsage( - input_tokens=input_text_tokens, - input_tokens_details=ImageUsageInputTokensDetails( - text_tokens=input_text_tokens, - image_tokens=0, - ), - output_tokens=output_tokens, - total_tokens=input_text_tokens + output_tokens, - prompt_tokens=input_text_tokens, - completion_tokens=output_tokens, - prompt_tokens_details={ - "text_tokens": input_text_tokens, - "image_tokens": 0, - }, - completion_tokens_details={ - "text_tokens": output_text_tokens, - "image_tokens": output_image_tokens, - }, - output_tokens_details={ - "text_tokens": output_text_tokens, - "image_tokens": output_image_tokens, - }, - ), - ) - - cost = gemini_image_generation_cost_calculator( - model=model, - image_response=image_response, - ) - - expected_cost = ( - input_text_tokens * model_info["input_cost_per_token"] - + output_text_tokens * model_info["output_cost_per_token"] - + output_image_tokens * model_info["output_cost_per_image_token"] - ) - all_output_as_image_cost = ( - input_text_tokens * model_info["input_cost_per_token"] - + (output_text_tokens + output_image_tokens) - * model_info["output_cost_per_image_token"] - ) - assert round(cost, 10) == round(expected_cost, 10) - assert cost != all_output_as_image_cost -def test_gemini_image_edit_cost_falls_back_to_flat_image_pricing(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - model = "gemini/gemini-3-pro-image-preview" - model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") - image_response = ImageResponse( - data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")] - ) - - cost = gemini_image_edit_cost_calculator( - model=model, - image_response=image_response, - ) - - assert cost == len(image_response.data or []) * model_info["output_cost_per_image"] def _image_response_with_web_search(web_search_requests): @@ -383,43 +223,8 @@ def _image_response_with_web_search(web_search_requests): return ImageResponse(data=[ImageObject(b64_json="img1")], usage=usage) -def test_gemini_image_generation_cost_adds_web_search_grounding(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - model = "gemini/gemini-3-pro-image-preview" - model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") - - grounded = gemini_image_generation_cost_calculator( - model=model, - image_response=_image_response_with_web_search(2), - ) - ungrounded = gemini_image_generation_cost_calculator( - model=model, - image_response=_image_response_with_web_search(None), - ) - - expected_web_search_cost = cost_per_web_search_request( - usage=_make_usage(2), model_info=model_info - ) - assert expected_web_search_cost > 0 - assert round(grounded - ungrounded, 10) == round(expected_web_search_cost, 10) -def test_gemini_image_generation_cost_no_web_search_when_absent(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - litellm.model_cost = litellm.get_model_cost_map(url="") - model = "gemini/gemini-3-pro-image-preview" - - cost_zero = gemini_image_generation_cost_calculator( - model=model, - image_response=_image_response_with_web_search(0), - ) - cost_none = gemini_image_generation_cost_calculator( - model=model, - image_response=_image_response_with_web_search(None), - ) - - assert cost_zero == cost_none @pytest.mark.parametrize( diff --git a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py index f1f1978b06f..b1d7e49fbac 100644 --- a/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py +++ b/tests/test_litellm/llms/github_copilot/test_github_copilot_transformation.py @@ -371,62 +371,8 @@ def test_x_initiator_header_system_only_messages(): assert headers["X-Initiator"] == "user" -def test_get_supported_openai_params_claude_model(): - """Test that Claude models with extended thinking support have thinking and reasoning parameters.""" - config = GithubCopilotConfig() - - # Test Claude 4 model supports thinking and reasoning_effort parameters - supported_params = config.get_supported_openai_params("claude-sonnet-4-20250514") - assert "thinking" in supported_params - assert "reasoning_effort" in supported_params - - # Test Claude 3-7 model supports thinking and reasoning_effort parameters - supported_params_claude37 = config.get_supported_openai_params( - "claude-3-7-sonnet-20250219" - ) - assert "thinking" in supported_params_claude37 - assert "reasoning_effort" in supported_params_claude37 - - # Test Claude 3.5 model does NOT support thinking parameters (no extended thinking) - supported_params_claude35 = config.get_supported_openai_params("claude-3.5-sonnet") - assert "thinking" not in supported_params_claude35 - assert "reasoning_effort" not in supported_params_claude35 - - # Test non-Claude model doesn't include thinking parameters but may include reasoning_effort - supported_params_gpt = config.get_supported_openai_params("gpt-4o") - assert "thinking" not in supported_params_gpt - # gpt-4o should NOT have reasoning_effort (not a reasoning model) - assert "reasoning_effort" not in supported_params_gpt - - # Test O-series reasoning models include reasoning_effort but not thinking - supported_params_o3 = config.get_supported_openai_params("o3-mini") - assert "thinking" not in supported_params_o3 - # o3-mini should have reasoning_effort (it's an O-series reasoning model) - assert "reasoning_effort" in supported_params_o3 -def test_get_supported_openai_params_case_insensitive(): - """Test that Claude model detection is case-insensitive for models with extended thinking.""" - config = GithubCopilotConfig() - - # Test uppercase Claude 4 model with full model name - supported_params_upper = config.get_supported_openai_params( - "CLAUDE-SONNET-4-20250514" - ) - assert "thinking" in supported_params_upper - assert "reasoning_effort" in supported_params_upper - - # Test mixed case Claude 3-7 model (has extended thinking) with full model name - supported_params_mixed = config.get_supported_openai_params( - "Claude-3-7-Sonnet-20250219" - ) - assert "thinking" in supported_params_mixed - assert "reasoning_effort" in supported_params_mixed - - # Test that Claude 3.5 models don't have thinking support (case insensitive) - supported_params_35 = config.get_supported_openai_params("CLAUDE-3.5-SONNET") - assert "thinking" not in supported_params_35 - assert "reasoning_effort" not in supported_params_35 def test_copilot_vision_request_header_with_image(): diff --git a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py b/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py index 4221954d787..7cd2b9e259c 100644 --- a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py @@ -416,3 +416,52 @@ async def test_async_realtime_ws_url_has_no_ssl(): # Verify ssl is None for ws:// URLs (the fix for issue #19222) assert called_kwargs["ssl"] is None + + +@pytest.mark.asyncio +async def test_async_realtime_upstream_handshake_refusal_sends_error_event_then_policy_close(): + from typing import cast + + import websockets + + from litellm.llms.openai.realtime.handler import OpenAIRealtime + from litellm.types.realtime import RealtimeErrorEvent + + handler = OpenAIRealtime() + model = "gpt-realtime" + + sent: list[str] = [] + closed: list[tuple[int, str | None]] = [] + + class RecordingClientWebSocket: + scope: dict[str, list[tuple[bytes, bytes]]] = {"headers": []} + + async def send_text(self, data: str) -> None: + sent.append(data) + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + closed.append((code, reason)) + + dummy_websocket = RecordingClientWebSocket() + dummy_logging_obj = MagicMock() + + refused = websockets.exceptions.InvalidStatus( + websockets.http11.Response(401, "Unauthorized", websockets.datastructures.Headers()) + ) + + with patch("websockets.connect", side_effect=refused): + await handler.async_realtime( # pyright: ignore[reportUnknownMemberType] # handler's websocket param is Any + model=model, + websocket=dummy_websocket, + logging_obj=dummy_logging_obj, + api_base="https://api.openai.com/", + api_key="bad-key", + query_params={"model": model}, + ) + + assert len(sent) == 1 + event = cast(RealtimeErrorEvent, json.loads(sent[0])) + assert event["type"] == "error" + assert event["error"]["type"] == "server_error" + assert "401" in event["error"]["message"] + assert closed and closed[0][0] == 1008 diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index 0adc7fa8d5f..41f5816600f 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -38,10 +38,6 @@ def test_gpt5_supports_reasoning_effort(config: OpenAIConfig): assert "reasoning_effort" in config.get_supported_openai_params(model="gpt-5-mini") -def test_gpt5_chat_does_not_support_reasoning_effort(config: OpenAIConfig): - assert "reasoning_effort" not in config.get_supported_openai_params( - model="gpt-5-chat-latest" - ) def test_gpt5_chat_supports_temperature(config: OpenAIConfig): @@ -174,10 +170,6 @@ def test_gpt5_codex_unsupported_params_drop(config: OpenAIConfig): assert param not in config.get_supported_openai_params(model="gpt-5-codex") -def test_gpt5_codex_supports_tool_choice(gpt5_config: OpenAIGPT5Config): - """Test that GPT-5-Codex supports tool_choice parameter.""" - supported_params = gpt5_config.get_supported_openai_params(model="gpt-5-codex") - assert "tool_choice" in supported_params def test_gpt5_codex_supports_function_calling(config: OpenAIConfig): @@ -246,14 +238,6 @@ def test_gpt5_1_reasoning_effort_none(config: OpenAIConfig): assert params["reasoning_effort"] == effort -def test_gpt5_1_codex_max_allows_reasoning_effort_xhigh(config: OpenAIConfig): - params = config.map_openai_params( - non_default_params={"reasoning_effort": "xhigh"}, - optional_params={}, - model="gpt-5.1-codex-max", - drop_params=False, - ) - assert params["reasoning_effort"] == "xhigh" def test_gpt5_rejects_reasoning_effort_xhigh_for_other_models(config: OpenAIConfig): diff --git a/tests/test_litellm/llms/openai_like/test_provider_affinity_forwarding.py b/tests/test_litellm/llms/openai_like/test_provider_affinity_forwarding.py new file mode 100644 index 00000000000..43101234e63 --- /dev/null +++ b/tests/test_litellm/llms/openai_like/test_provider_affinity_forwarding.py @@ -0,0 +1,389 @@ +import json +from unittest.mock import MagicMock + +import httpx +import pytest + +import litellm +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.openai_like import dynamic_config +from litellm.llms.openai_like.json_loader import JSONProviderRegistry, SimpleProviderConfig + + +def _provider(*, responses: bool = False) -> SimpleProviderConfig: + endpoints = ["/v1/chat/completions"] + if responses: + endpoints.append("/v1/responses") + return SimpleProviderConfig( + "db_only_provider", + { + "base_url": "https://db-only.example/v1", + "api_key_env": "DYNAMIC_PROVIDER_API_KEY", + "supported_endpoints": endpoints, + }, + ) + + +def _chat_response_payload(content: str = "dynamic response") -> dict[str, object]: + return { + "id": "chatcmpl-dynamic-provider", + "object": "chat.completion", + "created": 1234567890, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": content}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 2, "total_tokens": 4}, + } + + +def _responses_payload() -> dict[str, object]: + return { + "id": "resp_dynamic_provider", + "object": "response", + "created_at": 1234567890, + "status": "completed", + "model": "test-model", + "output": [], + "parallel_tool_calls": True, + "usage": {"input_tokens": 2, "output_tokens": 2, "total_tokens": 4}, + "error": None, + } + + +@pytest.fixture(autouse=True) +def _isolate_registry_state(): + original_providers = dict(JSONProviderRegistry._providers) + dynamic_config._responses_config_cache.clear() + yield + JSONProviderRegistry._providers = original_providers + dynamic_config._responses_config_cache.clear() + + +def test_dynamic_provider_receives_affinity_header_for_chat(): + from openai import OpenAI + + JSONProviderRegistry._providers = {"db_only_provider": _provider()} + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_chat_response_payload()) + + client = OpenAI( + api_key="test-key", + base_url="https://db-only.example/v1", + http_client=httpx.Client(transport=httpx.MockTransport(respond)), + ) + try: + litellm.completion( + model="db_only_provider/test-model", + messages=[{"role": "user", "content": "hello"}], + api_key="test-key", + client=client, + extra_headers={"X-Customer-Header": "customer-value"}, + litellm_session_id="session-sync", + provider_affinity_header="X-Conversation-Id", + ) + finally: + client.close() + + assert requests[0].headers["x-conversation-id"] == "session-sync" + assert requests[0].headers["x-customer-header"] == "customer-value" + + +def test_dynamic_provider_does_not_use_trace_id_for_chat_affinity(): + from openai import OpenAI + + JSONProviderRegistry._providers = {"db_only_provider": _provider()} + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_chat_response_payload()) + + client = OpenAI( + api_key="test-key", + base_url="https://db-only.example/v1", + http_client=httpx.Client(transport=httpx.MockTransport(respond)), + ) + try: + litellm.completion( + model="db_only_provider/test-model", + messages=[{"role": "user", "content": "hello"}], + api_key="test-key", + client=client, + metadata={"trace_id": "per-request-trace"}, + provider_affinity_header="X-Conversation-Id", + ) + finally: + client.close() + + assert "x-conversation-id" not in requests[0].headers + + +@pytest.mark.asyncio +async def test_dynamic_provider_receives_affinity_header_for_async_chat(): + from openai import AsyncOpenAI + + JSONProviderRegistry._providers = {"db_only_provider": _provider()} + requests: list[httpx.Request] = [] + + async def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_chat_response_payload("async response")) + + client = AsyncOpenAI( + api_key="test-key", + base_url="https://db-only.example/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(respond)), + ) + try: + response = await litellm.acompletion( + model="db_only_provider/test-model", + messages=[{"role": "user", "content": "hello"}], + api_key="test-key", + client=client, + litellm_session_id="session-async", + provider_affinity_header="X-Conversation-Id", + ) + finally: + await client.close() + + assert response.choices[0].message.content == "async response" + assert requests[0].headers["x-conversation-id"] == "session-async" + + +def test_dynamic_provider_receives_affinity_header_for_streaming_chat(): + from openai import OpenAI + + JSONProviderRegistry._providers = {"db_only_provider": _provider()} + chunks = [ + { + "id": "chatcmpl-dynamic-stream", + "object": "chat.completion.chunk", + "created": 1234567890, + "model": "test-model", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "streamed"}, + "finish_reason": None, + } + ], + }, + { + "id": "chatcmpl-dynamic-stream", + "object": "chat.completion.chunk", + "created": 1234567890, + "model": "test-model", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + ] + stream_body = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n" + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, content=stream_body, headers={"content-type": "text/event-stream"}) + + client = OpenAI( + api_key="test-key", + base_url="https://db-only.example/v1", + http_client=httpx.Client(transport=httpx.MockTransport(respond)), + ) + try: + response_chunks = list( + litellm.completion( + model="db_only_provider/test-model", + messages=[{"role": "user", "content": "hello"}], + api_key="test-key", + client=client, + stream=True, + litellm_session_id="session-stream", + provider_affinity_header="X-Conversation-Id", + ) + ) + finally: + client.close() + + assert any(chunk.choices[0].delta.content == "streamed" for chunk in response_chunks) + assert requests[0].headers["x-conversation-id"] == "session-stream" + + +def test_dynamic_provider_receives_affinity_header_for_responses(): + JSONProviderRegistry._providers = {"db_only_provider": _provider(responses=True)} + logging_obj = MagicMock() + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_responses_payload()) + + http_client = httpx.Client(transport=httpx.MockTransport(respond)) + try: + litellm.responses( + model="db_only_provider/test-model", + input="hello", + api_key="test-key", + litellm_session_id="session-responses", + provider_affinity_header="X-Conversation-Id", + litellm_logging_obj=logging_obj, + client=HTTPHandler(client=http_client), + ) + finally: + http_client.close() + + assert requests[0].headers["X-Conversation-Id"] == "session-responses" + assert ( + logging_obj.update_from_kwargs.call_args.kwargs["litellm_params"]["provider_affinity_header"] + == "X-Conversation-Id" + ) + + +def test_dynamic_provider_uses_metadata_session_id_for_responses(): + JSONProviderRegistry._providers = {"db_only_provider": _provider(responses=True)} + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_responses_payload()) + + http_client = httpx.Client(transport=httpx.MockTransport(respond)) + try: + litellm.responses( + model="db_only_provider/test-model", + input="hello", + api_key="test-key", + metadata={"session_id": "session-from-metadata"}, + provider_affinity_header="X-Conversation-Id", + litellm_logging_obj=MagicMock(), + client=HTTPHandler(client=http_client), + ) + finally: + http_client.close() + + assert requests[0].headers["X-Conversation-Id"] == "session-from-metadata" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +async def test_dynamic_provider_receives_affinity_header_for_async_responses(stream: bool): + JSONProviderRegistry._providers = {"db_only_provider": _provider(responses=True)} + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_responses_payload()) + + client = AsyncHTTPHandler() + await client.close() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(respond)) + try: + response = await litellm.aresponses( + model="db_only_provider/test-model", + input="hello", + api_key="test-key", + stream=stream, + litellm_session_id="session-async-responses", + provider_affinity_header="X-Conversation-Id", + client=client, + ) + finally: + await client.client.aclose() + + assert requests[0].headers["X-Conversation-Id"] == "session-async-responses" + if stream: + assert hasattr(response, "__aiter__") + else: + assert getattr(response, "model", None) == "test-model" + + +def test_control_characters_in_session_id_are_a_bad_request_for_chat(): + from openai import OpenAI + + JSONProviderRegistry._providers = {"db_only_provider": _provider()} + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_chat_response_payload()) + + client = OpenAI( + api_key="test-key", + base_url="https://db-only.example/v1", + http_client=httpx.Client(transport=httpx.MockTransport(respond)), + ) + try: + with pytest.raises(litellm.BadRequestError, match="HTTP header control characters"): + litellm.completion( + model="db_only_provider/test-model", + messages=[{"role": "user", "content": "hello"}], + api_key="test-key", + client=client, + litellm_session_id="session\nsplit", + provider_affinity_header="X-Conversation-Id", + ) + finally: + client.close() + + assert requests == [] + + +def test_control_characters_in_session_id_are_a_bad_request_for_responses(): + JSONProviderRegistry._providers = {"db_only_provider": _provider(responses=True)} + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_responses_payload()) + + http_client = httpx.Client(transport=httpx.MockTransport(respond)) + try: + with pytest.raises(litellm.BadRequestError, match="HTTP header control characters"): + litellm.responses( + model="db_only_provider/test-model", + input="hello", + api_key="test-key", + litellm_session_id="session\nsplit", + provider_affinity_header="X-Conversation-Id", + litellm_logging_obj=MagicMock(), + client=HTTPHandler(client=http_client), + ) + finally: + http_client.close() + + assert requests == [] + + +def test_builtin_provider_receives_affinity_header(): + from openai import OpenAI + + requests: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_chat_response_payload()) + + client = OpenAI( + api_key="test-key", + base_url="https://api.openai.com/v1", + http_client=httpx.Client(transport=httpx.MockTransport(respond)), + ) + try: + litellm.completion( + model="openai/test-model", + messages=[{"role": "user", "content": "hello"}], + api_key="test-key", + client=client, + litellm_session_id="session-builtin", + provider_affinity_header="X-Conversation-Id", + ) + finally: + client.close() + + assert requests[0].headers["X-Conversation-Id"] == "session-builtin" diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 6c818016c87..8b0662de3f9 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -2435,10 +2435,16 @@ def test_is_gemini_3_or_newer(): VertexGeminiConfig, ) - # Gemini 3 models + # Gemini 3+ models assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-3-pro-preview") == True assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-3-flash") == True assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-3-pro") == True + assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-3.1-pro-preview") == True + assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-test-id-bla") == True + assert VertexGeminiConfig._is_gemini_3_or_newer("test-id-bla") == True + assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-flash-latest") == True + assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-flash-lite-latest") == True + assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-pro-latest") == True assert ( VertexGeminiConfig._is_gemini_3_or_newer("vertex_ai/gemini-3-pro-preview") == True @@ -2453,11 +2459,53 @@ def test_is_gemini_3_or_newer(): assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-2.0-flash") == False assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-1.5-pro") == False assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-pro") == False + assert VertexGeminiConfig._is_gemini_3_or_newer("gemini-flash") == False # Edge cases assert VertexGeminiConfig._is_gemini_3_or_newer("") == False +@pytest.mark.parametrize( + "model", + [ + "gemini-3.1-pro-preview", + "gemini-3-flash", + "gemini-test-id-bla", + "test-id-bla", + ], +) +def test_gemini_3_reasoning_effort_maps_to_thinking_level(model: str): + """Test that reasoning_effort maps to thinkingLevel and default temperature=1.0""" + from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + vertex_cfg = VertexGeminiConfig() + studio_cfg = GoogleAIStudioGeminiConfig() + + for cfg in (vertex_cfg, studio_cfg): + supported = cfg.get_supported_openai_params(model) + assert "reasoning_effort" in supported + assert "thinking" in supported + + for effort in ("low", "medium", "high"): + mapped = vertex_cfg.map_openai_params( + non_default_params={"reasoning_effort": effort}, + optional_params={}, + model=model, + drop_params=False, + ) + assert mapped["thinkingConfig"] == { + "thinkingLevel": effort, + "includeThoughts": True, + } + assert mapped["temperature"] == 1.0 + assert "thinkingBudget" not in mapped["thinkingConfig"] + + + + def _tool_call_messages(tool_call_id: str): return [ {"role": "user", "content": "hi"}, @@ -2643,7 +2691,7 @@ def test_reasoning_effort_maps_to_thinking_level_gemini_3(): assert result["thinkingConfig"]["thinkingLevel"] == "low" assert result["thinkingConfig"]["includeThoughts"] is True - # Test medium -> high + includeThoughts=True (medium not available yet) + # Test medium -> medium + includeThoughts=True optional_params = {} non_default_params = {"reasoning_effort": "medium"} result = v.map_openai_params( @@ -2652,7 +2700,7 @@ def test_reasoning_effort_maps_to_thinking_level_gemini_3(): model=model, drop_params=False, ) - assert result["thinkingConfig"]["thinkingLevel"] == "high" + assert result["thinkingConfig"]["thinkingLevel"] == "medium" assert result["thinkingConfig"]["includeThoughts"] is True # Test high -> high + includeThoughts=True @@ -2853,7 +2901,7 @@ def test_reasoning_effort_dict_format_gemini_3(): model=model, drop_params=False, ) - assert result["thinkingConfig"]["thinkingLevel"] == "high" + assert result["thinkingConfig"]["thinkingLevel"] == "medium" assert result["thinkingConfig"]["includeThoughts"] is True # Test dict format without effort key - no thinkingConfig should be set @@ -5900,8 +5948,8 @@ def test_calculate_web_search_requests_counts_unique_queries(): @pytest.mark.parametrize("custom_llm_provider", ["gemini", "vertex_ai"]) @pytest.mark.parametrize( "model", - ["gemini-2.5-flash", "gemini-3-pro-preview"], - ids=["thinking_budget_mapper", "thinking_level_mapper"], + ["gemini-2.5-flash"], + ids=["thinking_budget_mapper"], ) @pytest.mark.parametrize("reasoning_effort", ["banana", "xhigh"]) def test_invalid_reasoning_effort_is_a_400_not_a_500(custom_llm_provider, model, reasoning_effort): diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py index 58e7529309a..4199e57d2f9 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py @@ -238,31 +238,3 @@ def test_audio_predict_response_supports_bytes_base64_encoded( assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.06) -def test_image_predict_response_is_not_billed_as_audio( - local_model_cost_map: None, -) -> None: - logging_obj = MagicMock() - logging_obj.model_call_details = {} - response = httpx.Response( - status_code=200, - json={"predictions": [{"bytesBase64Encoded": "frame", "mimeType": "image/png"}]}, - ) - - result = VertexPassthroughLoggingHandler.vertex_passthrough_handler( - httpx_response=response, - logging_obj=logging_obj, - url_route=( - "/v1/projects/test/locations/us-central1/publishers/google/models/imagen-4.0-generate-001:predict" - ), - result=response.text, - start_time=datetime.now(), - end_time=datetime.now(), - cache_hit=False, - request_body={"instances": [{"prompt": "a red cube"}]}, - ) - - assert isinstance(result["result"], litellm.ImageResponse) - assert logging_obj.call_type == PassthroughCallTypes.passthrough_image_generation.value - assert result["kwargs"]["response_cost"] == pytest.approx( - litellm.model_cost["vertex_ai/imagen-4.0-generate-001"]["output_cost_per_image"] - ) diff --git a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py index a25ed585ecc..5c168d84766 100644 --- a/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py +++ b/tests/test_litellm/llms/wandb/test_wandb_chat_transformation.py @@ -37,10 +37,6 @@ WANDB_REASONING_MODELS: Final = ( "Qwen/Qwen3.5-35B-A3B", "zai-org/GLM-5.2", "moonshotai/Kimi-K2.5", - "MiniMaxAI/MiniMax-M2.5", - "zai-org/GLM-4.5", - "Qwen/Qwen3-235B-A22B-Thinking-2507", - "deepseek-ai/DeepSeek-R1-0528", ) diff --git a/tests/test_litellm/llms/xai/test_xai_cost_calculator.py b/tests/test_litellm/llms/xai/test_xai_cost_calculator.py index cf3bc73a225..6065f1053e7 100644 --- a/tests/test_litellm/llms/xai/test_xai_cost_calculator.py +++ b/tests/test_litellm/llms/xai/test_xai_cost_calculator.py @@ -152,83 +152,10 @@ class TestXAICostCalculator: setattr(reported, "server_side_tool_usage_details", {"web_search_calls": 3}) assert get_cost_for_web_search_request("xai", reported, {}) == 0.0 - def test_no_reported_cost_falls_back_to_token_math(self): - """Absent the provider figure, nothing changes for existing callers.""" - usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300) - prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) - assert prompt_cost > 0.0 - assert completion_cost > 0.0 - def test_malformed_reported_cost_falls_back_to_token_math(self): - """A junk value must not fail the request, fall back to calculating.""" - usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300) - setattr(usage, "cost", "not-a-number") - prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) - - assert prompt_cost > 0.0 - assert completion_cost > 0.0 - - def test_boolean_reported_cost_falls_back_to_token_math(self): - """True is an int in python and would otherwise be billed as $1.""" - usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300) - setattr(usage, "cost", True) - - prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) - - assert prompt_cost > 0.0 - assert completion_cost > 0.0 - assert completion_cost != 1.0 - - def test_negative_reported_cost_is_rejected(self): - """A negative amount must never reach spend tracking. - - A caller who can set api_base controls the response body, so trusting a - negative figure would let them subtract from their own recorded spend and - slip past a budget. Fall back to token pricing instead, and keep charging - the web search surcharge, since no trustworthy total was reported. - """ - usage = Usage( - prompt_tokens=100, - completion_tokens=200, - total_tokens=300, - cost=-0.0037756, - ) - setattr(usage, "server_side_tool_usage_details", {"web_search_calls": 3}) - - prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) - - assert prompt_cost > 0.0 - assert completion_cost > 0.0 - assert cost_per_web_search_request(usage=usage, model_info={}) > 0.0 - - def test_non_finite_reported_cost_is_rejected(self): - """NaN compares false against every budget threshold. - - Usage stores a provider supplied cost without validating it, so a caller who - controls the response body could report NaN and leave spend >= max_budget - false for the life of the key rather than mispricing one request. The - infinities are refused alongside it. Fall back to token pricing and keep - charging the web search surcharge, since no trustworthy total was reported. - """ - for reported_cost in (float("nan"), float("inf"), float("-inf")): - usage = Usage( - prompt_tokens=100, - completion_tokens=200, - total_tokens=300, - cost=reported_cost, - ) - setattr(usage, "server_side_tool_usage_details", {"web_search_calls": 3}) - - prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) - - assert math.isfinite(prompt_cost), reported_cost - assert math.isfinite(completion_cost), reported_cost - assert prompt_cost > 0.0, reported_cost - assert completion_cost > 0.0, reported_cost - assert cost_per_web_search_request(usage=usage, model_info={}) > 0.0, reported_cost def test_zero_reported_cost_is_honoured(self): """A reported zero is a real answer, not a missing value.""" diff --git a/tests/test_litellm/llms/xai/test_xai_key_fallback.py b/tests/test_litellm/llms/xai/test_xai_key_fallback.py index 092e4951547..cbc507c5ee3 100644 --- a/tests/test_litellm/llms/xai/test_xai_key_fallback.py +++ b/tests/test_litellm/llms/xai/test_xai_key_fallback.py @@ -12,6 +12,9 @@ from litellm.types.router import GenericLiteLLMParams class FakeLogging: + def __init__(self) -> None: + self.litellm_params: dict = {} + def update_from_kwargs(self, **kwargs): pass diff --git a/tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py b/tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py deleted file mode 100644 index 83e8925f70b..00000000000 --- a/tests/test_litellm/llms/xai/test_xai_redirected_slug_pricing.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -xAI retired eight slugs on 2026-05-15 but kept them resolvable: chat slugs redirect to -grok-4.3 and bill at grok-4.3's rates, while the grok-code-fast slugs are aliases of -grok-build-0.1 and bill at its rates, so the registry must price them that way or spend -tracking is wrong. The grok-3-beta, grok-3-fast, grok-3-mini, and grok-4-1-fast slugs -are absent from /v1/language-models and resolve to grok-4.3 the same way (the chat -response names grok-4.3 as the served model), so they carry grok-4.3's rates too. -https://docs.x.ai/developers/migration/may-15-retirement -https://docs.x.ai/developers/models/grok-build-0.1 -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -REPO_ROOT = Path(__file__).parents[4] -PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json" -BACKUP_PRICES_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json" -MAP_PATHS = (PRICES_PATH, BACKUP_PRICES_PATH) - -REDIRECT_TARGET = "xai/grok-4.3" -GROK_3_MINI_SLUGS = ( - "xai/grok-3-mini", - "xai/grok-3-mini-beta", - "xai/grok-3-mini-fast", - "xai/grok-3-mini-fast-beta", - "xai/grok-3-mini-fast-latest", - "xai/grok-3-mini-latest", -) -REDIRECTED_SLUGS = ( - "xai/grok-3", - "xai/grok-3-beta", - "xai/grok-3-fast-beta", - "xai/grok-3-fast-latest", - "xai/grok-3-latest", - *GROK_3_MINI_SLUGS, - "xai/grok-4", - "xai/grok-4-0709", - "xai/grok-4-1-fast", - "xai/grok-4-1-fast-non-reasoning", - "xai/grok-4-1-fast-non-reasoning-latest", - "xai/grok-4-1-fast-reasoning", - "xai/grok-4-1-fast-reasoning-latest", - "xai/grok-4-fast-non-reasoning", - "xai/grok-4-fast-reasoning", - "xai/grok-4-latest", -) -CODE_REDIRECT_TARGET = "xai/grok-build-0.1" -CODE_SLUGS = ( - "xai/grok-code-fast", - "xai/grok-code-fast-1", - "xai/grok-code-fast-1-0825", -) -BASE_COST_FIELDS = ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost") -TIER_COST_FIELDS = ( - "input_cost_per_token_above_200k_tokens", - "output_cost_per_token_above_200k_tokens", - "cache_read_input_token_cost_above_200k_tokens", -) - - -@pytest.fixture(scope="module", params=[p.name for p in MAP_PATHS]) -def cost_map(request: pytest.FixtureRequest) -> dict: - path = next(p for p in MAP_PATHS if p.name == request.param) - return json.loads(path.read_text(encoding="utf-8")) - - -@pytest.mark.parametrize("slug", REDIRECTED_SLUGS) -def test_redirected_slug_bills_at_the_target_rate(cost_map: dict, slug: str): - target = cost_map[REDIRECT_TARGET] - entry = cost_map[slug] - for field in BASE_COST_FIELDS: - assert entry[field] == target[field], field - - -@pytest.mark.parametrize("slug", CODE_SLUGS) -def test_code_slug_bills_at_grok_build_rate(cost_map: dict, slug: str): - """grok-code-fast* are aliases of grok-build-0.1, not grok-4.3 redirects.""" - target = cost_map[CODE_REDIRECT_TARGET] - entry = cost_map[slug] - for field in (*BASE_COST_FIELDS, *TIER_COST_FIELDS): - assert entry[field] == target[field], field - - -@pytest.mark.parametrize("slug", REDIRECTED_SLUGS) -def test_redirected_slug_carries_the_target_tier_rates(cost_map: dict, slug: str): - """The request executes as grok-4.3, so it is tiered at grok-4.3's 200k boundary.""" - target = cost_map[REDIRECT_TARGET] - entry = cost_map[slug] - for field in TIER_COST_FIELDS: - assert entry[field] == target[field], field - assert {k for k in entry if "_above_" in k} == {k for k in target if "_above_" in k} - - -def test_both_cost_maps_agree_on_the_redirected_slugs(): - prices = json.loads(PRICES_PATH.read_text(encoding="utf-8")) - backup = json.loads(BACKUP_PRICES_PATH.read_text(encoding="utf-8")) - for slug in (*REDIRECTED_SLUGS, *CODE_SLUGS, REDIRECT_TARGET, CODE_REDIRECT_TARGET): - assert prices[slug] == backup[slug], slug diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 5c885a3168d..0012be23428 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -3057,6 +3057,11 @@ class TestMCPDelegateAuthToUpstream: ) cases = [ + ("/mcp/sse", []), + ("/mcp/sse/", []), + ("/mcp/sse/messages", []), + ("/mcp/sse/messages/", []), + ("/sse/mcp", ["sse"]), # Single server, single segment. ("/mcp/foo", ["foo"]), # Server name with one embedded slash (two segments). @@ -5936,12 +5941,13 @@ class TestMCPDcrBridgeDelegateAdmission: @staticmethod def _wrapped_user_lookup_error(original: BaseException) -> ValueError: - """Reproduce get_user_object's real exception contract (litellm/proxy/auth/auth_checks.py): it - catches every DB failure in a broad ``except`` and re-raises a bare ``ValueError``, so the - original error (a missing-user Exception or a real outage) survives only as ``__context__``. - Injecting a raw ConnectionError/Exception instead would exercise a shape production never - produces and let a chain-blind outage classifier pass. That wrapping fidelity is itself pinned by - test_get_user_object_wraps_db_outage_as_valueerror_preserving_context in test_auth_checks.""" + """Reproduce get_user_object's exception contract (litellm/proxy/auth/auth_checks.py): a read + failure that is not a database outage is re-raised as a bare ``ValueError`` with the original + error only as ``__context__``, while an outage propagates raw (pinned by + test_get_user_object_surfaces_a_db_outage_as_503_not_as_a_missing_user and + test_get_user_object_still_reports_a_non_outage_read_failure_as_a_missing_user in + test_auth_checks). The wrapped shape is the harder one for the outage classifier, so injecting + it here keeps a chain-blind classifier from passing.""" try: raise original except BaseException: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index b715fe67e20..ea82c9b069d 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -1147,109 +1147,39 @@ async def test_mcp_read_resource_success(): assert result is read_result -def test_normalize_resource_contents_passes_metadata(): - """Test that _normalize_resource_contents preserves meta from ResourceContents (MCP 1.26.0+).""" - try: - from litellm.proxy._experimental.mcp_server.server import ( - _normalize_resource_contents, - ) - except ImportError: - pytest.skip("MCP server not available") +@pytest.mark.asyncio +@pytest.mark.parametrize( + "kind,metadata", + (("text", {"version": "1.0", "source": "test"}), ("blob", {"encoding": "base64"}), ("text", {}), ("text", None)), +) +async def test_read_resource_preserves_content_metadata(_mcp_request_ctx, kind, metadata): + from mcp.types import ReadResourceRequestParams + from litellm.proxy._experimental.mcp_server import operations, server - meta = {"version": "1.0", "source": "test"} - contents = [ - TextResourceContents( - uri="https://example.com/resource", - text="hello world", - mimeType="text/plain", - meta=meta, - ) - ] + uri: Final = "https://example.com/resource" + caller: Final = UserAPIKeyAuth(user_id="resource-caller") + upstream_server: Final = MCPServer(server_id="catalog", name="catalog", transport=MCPTransport.http) + content: Final = ( + TextResourceContents(uri=uri, text="hello world", mimeType="text/plain", meta=metadata) + if kind == "text" + else BlobResourceContents(uri=uri, blob="aGVsbG8=", mimeType="image/png", meta=metadata) + ) + with ( + patch.object(server, "get_or_extract_auth_context", AsyncMock(return_value=(caller, None, ["catalog"], None, None, None, None))), + patch.object(operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[upstream_server])), + patch.object(operations.global_mcp_server_manager, "read_resource_from_server", AsyncMock(return_value=ReadResourceResult(contents=[content]))), + ): + result: Final = await server.read_resource(_mcp_request_ctx(), ReadResourceRequestParams(uri=uri)) - result = _normalize_resource_contents(contents) - - assert len(result) == 1 - assert result[0].content == "hello world" - assert result[0].mime_type == "text/plain" - assert result[0].meta == meta - - -def test_normalize_resource_contents_blob_with_metadata(): - """Test that _normalize_resource_contents preserves meta for BlobResourceContents.""" - try: - from litellm.proxy._experimental.mcp_server.server import ( - _normalize_resource_contents, - ) - except ImportError: - pytest.skip("MCP server not available") - - meta = {"encoding": "base64"} - contents = [ - BlobResourceContents( - uri="https://example.com/image.png", - blob="aGVsbG8=", - mimeType="image/png", - meta=meta, - ) - ] - - result = _normalize_resource_contents(contents) - - assert len(result) == 1 - assert result[0].content == "aGVsbG8=" - assert result[0].mime_type == "image/png" - assert result[0].meta == meta - - -def test_normalize_resource_contents_preserves_empty_metadata(): - """Test that empty dict meta is preserved (truthiness bug fix).""" - try: - from litellm.proxy._experimental.mcp_server.server import ( - _normalize_resource_contents, - ) - except ImportError: - pytest.skip("MCP server not available") - - empty_meta: dict = {} - contents = [ - TextResourceContents( - uri="https://example.com/resource", - text="hi", - mimeType="text/plain", - meta=empty_meta, - ) - ] - - result = _normalize_resource_contents(contents) - - assert len(result) == 1 - assert result[0].meta == empty_meta - assert result[0].meta is not None - assert result[0].meta == {} - - -def test_normalize_resource_contents_without_metadata(): - """Test that _normalize_resource_contents works when meta is absent (backward compat).""" - try: - from litellm.proxy._experimental.mcp_server.server import ( - _normalize_resource_contents, - ) - except ImportError: - pytest.skip("MCP server not available") - - contents = [ - TextResourceContents( - uri="https://example.com/resource", - text="hello", - mimeType="text/plain", - ) - ] - - result = _normalize_resource_contents(contents) - - assert len(result) == 1 - assert result[0].content == "hello" - assert result[0].meta is None + assert result.model_dump(mode="json", by_alias=True, exclude_none=True) == { + "cacheScope": "private", "resultType": "complete", "ttlMs": 0, + "contents": [{ + "uri": uri, + "mimeType": "text/plain" if kind == "text" else "image/png", + "text" if kind == "text" else "blob": "hello world" if kind == "text" else "aGVsbG8=", + **({"_meta": metadata} if metadata is not None else {}), + }], + } @pytest.mark.asyncio @@ -1859,14 +1789,12 @@ async def test_concurrent_initialize_session_managers(): original_initialized = mcp_server._SESSION_MANAGERS_INITIALIZED original_session_cm = mcp_server._session_manager_cm original_stateful_cm = mcp_server._session_manager_stateful_cm - original_sse_cm = mcp_server._sse_session_manager_cm original_cleanup_task = mcp_server._stateful_auth_context_cleanup_task try: mcp_server._SESSION_MANAGERS_INITIALIZED = False mcp_server._session_manager_cm = None mcp_server._session_manager_stateful_cm = None - mcp_server._sse_session_manager_cm = None # Create mock context managers for all three session managers mock_cm_stateless = AsyncMock() @@ -1877,10 +1805,6 @@ async def test_concurrent_initialize_session_managers(): mock_cm_stateful.__aenter__ = AsyncMock() mock_cm_stateful.__aexit__ = AsyncMock() - mock_cm_sse = AsyncMock() - mock_cm_sse.__aenter__ = AsyncMock() - mock_cm_sse.__aexit__ = AsyncMock() - with ( patch.object( mcp_server.session_manager_stateless, @@ -1892,11 +1816,6 @@ async def test_concurrent_initialize_session_managers(): "run", return_value=mock_cm_stateful, ) as mock_stateful_run, - patch.object( - mcp_server.sse_session_manager, - "run", - return_value=mock_cm_sse, - ) as mock_sse_run, patch("litellm.proxy._experimental.mcp_server.operations.verbose_logger"), ): # Create multiple concurrent tasks that call initialize_session_managers @@ -1918,10 +1837,6 @@ async def test_concurrent_initialize_session_managers(): assert mock_stateful_run.call_count == 1, ( f"Expected 1 call to session_manager_stateful.run(), got {mock_stateful_run.call_count}" ) - assert mock_sse_run.call_count == 1, ( - f"Expected 1 call to sse_session_manager.run(), got {mock_sse_run.call_count}" - ) - # The context managers should only be entered once each assert mock_cm_stateless.__aenter__.call_count == 1, ( f"Expected 1 call to stateless __aenter__, got {mock_cm_stateless.__aenter__.call_count}" @@ -1929,10 +1844,6 @@ async def test_concurrent_initialize_session_managers(): assert mock_cm_stateful.__aenter__.call_count == 1, ( f"Expected 1 call to stateful __aenter__, got {mock_cm_stateful.__aenter__.call_count}" ) - assert mock_cm_sse.__aenter__.call_count == 1, ( - f"Expected 1 call to sse __aenter__, got {mock_cm_sse.__aenter__.call_count}" - ) - # State should be properly set assert mcp_server._SESSION_MANAGERS_INITIALIZED is True @@ -1948,7 +1859,6 @@ async def test_concurrent_initialize_session_managers(): mcp_server._SESSION_MANAGERS_INITIALIZED = original_initialized mcp_server._session_manager_cm = original_session_cm mcp_server._session_manager_stateful_cm = original_stateful_cm - mcp_server._sse_session_manager_cm = original_sse_cm mcp_server._stateful_auth_context_cleanup_task = original_cleanup_task @@ -2263,7 +2173,7 @@ async def test_sse_endpoint_applies_the_same_client_allowlist( new_callable=AsyncMock, ), patch.object( # test-quality-ok: SSE manager is a module singleton; the downstream call is the observable - mcp_module.sse_session_manager, "handle_request", side_effect=handle_request + mcp_module.sse, "handle_post_message", side_effect=handle_request ), ): if admitted: @@ -6651,17 +6561,22 @@ class TestGatewayCreateInitializationOptions: ) captured = {} - async def record_request(scope, receive, send): + @contextlib.asynccontextmanager + async def connect_sse(scope, receive, send): + yield (None, None) + + async def record_request(read_stream, write_stream, options): captured["server_name"] = server.create_initialization_options().server_name scope = { "type": "http", - "method": "POST", + "method": "GET", "path": "/mcp/grafana", "headers": [], } with ( + patch.object(mcp_server.sse, "connect_sse", connect_sse), patch( "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", new_callable=AsyncMock, @@ -6697,8 +6612,8 @@ class TestGatewayCreateInitializationOptions: True, ), patch.object( - mcp_server.sse_session_manager, - "handle_request", + mcp_server.server, + "run", side_effect=record_request, ), ): @@ -8840,7 +8755,7 @@ def _call_tool_result(is_error: bool, text: str) -> CallToolResult: def _mock_mcp_logging_obj() -> MagicMock: logging_obj = MagicMock() logging_obj.model_call_details = {} - logging_obj.async_post_mcp_tool_call_hook = AsyncMock() + logging_obj.async_post_mcp_tool_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response_obj"]) logging_obj.async_success_handler = AsyncMock() logging_obj.async_failure_handler = AsyncMock() return logging_obj @@ -8942,6 +8857,64 @@ async def test_fire_mcp_tool_call_logging_success_path_unchanged(): proxy_logging_mock.post_call_failure_hook.assert_not_awaited() +@pytest.mark.asyncio +async def test_fire_mcp_tool_call_logging_applies_hook_content(): + from litellm.integrations.custom_logger import CustomLogger + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy._experimental.mcp_server.server import ( + _fire_mcp_tool_call_logging, + ) + from litellm.types.mcp import MCPPostCallResponseObject + + class RedactingLogger(CustomLogger): + async def async_post_mcp_tool_call_hook( + self, + kwargs: dict[str, object], + response_obj: MCPPostCallResponseObject, + start_time: datetime, + end_time: datetime, + ) -> MCPPostCallResponseObject: + assert isinstance(response_obj.mcp_tool_call_response, list) + assert isinstance(response_obj.mcp_tool_call_response[0], TextContent) + response_obj.mcp_tool_call_response = [TextContent(type="text", text="[REDACTED]")] + return response_obj + + logging_obj = Logging( + model="MCP: weather/get_forecast", + messages=[{"role": "user", "content": "tool call"}], + stream=False, + call_type="call_mcp_tool", + start_time=datetime.now(), + litellm_call_id="test-mcp-hook-content", + function_id="test-fn", + dynamic_success_callbacks=[RedactingLogger()], + ) + proxy_logging_mock = _mock_mcp_proxy_logging() + result = CallToolResult( + content=[TextContent(type="text", text="SECRET-1234")], + structuredContent={"result": "SECRET-1234"}, + isError=False, + ) + + with patch( # test-quality-ok: [TQ008] inject proxy logging collaborator + "litellm.proxy.proxy_server.proxy_logging_obj", + proxy_logging_mock, + ): + hooked_result = await _fire_mcp_tool_call_logging( + logging_obj=logging_obj, + result=result, + start_time=datetime.now(), + end_time=datetime.now(), + user_api_key_auth=UserAPIKeyAuth(api_key="test-key", user_id="test-user"), + request_data={}, + ) + + assert isinstance(hooked_result.content[0], TextContent) + assert hooked_result.content[0].text == "[REDACTED]" + assert hooked_result.structured_content is None + assert hooked_result.is_error is True + + @pytest.mark.asyncio async def test_fire_mcp_tool_call_logging_iserror_without_auth_skips_failure_hook(): """Without a UserAPIKeyAuth the failure handlers still fire but the proxy @@ -8956,7 +8929,7 @@ async def test_fire_mcp_tool_call_logging_iserror_without_auth_skips_failure_hoo with patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_mock): await _fire_mcp_tool_call_logging( logging_obj=logging_obj, - result={"isError": True, "content": [{"type": "text", "text": "denied"}]}, + result=_call_tool_result(True, "denied"), start_time=datetime.now(), end_time=datetime.now(), ) @@ -10260,7 +10233,10 @@ async def test_active_request_ctx_var_feeds_auth_resolution_recording(_mcp_reque ("1999-01-01", True), ], ) -async def test_streamable_http_rejects_modern_protocol_version(header_value: str, expected_rejected: bool) -> None: +@pytest.mark.parametrize("handler", ("handle_streamable_http_mcp", "handle_sse_mcp")) +async def test_streamable_http_rejects_modern_protocol_version( + header_value: str, expected_rejected: bool, handler: str +) -> None: from litellm.proxy._experimental.mcp_server import server as mcp_module from litellm.proxy._experimental.mcp_server.server import unsupported_protocol_version @@ -10283,7 +10259,7 @@ async def test_streamable_http_rejects_modern_protocol_version(header_value: str async def send(message: Message) -> None: sent.append(message) - await mcp_module.handle_streamable_http_mcp(scope, receive, send) + await getattr(mcp_module, handler)(scope, receive, send) start = next(m for m in sent if m["type"] == "http.response.start") assert start["status"] == 400 @@ -10333,3 +10309,124 @@ async def test_tool_listing_preserves_permission_denial_when_failure_logging_fai logger.post_call_failure_hook.assert_awaited_once() assert logger.post_call_failure_hook.await_args.kwargs["original_exception"] is denial assert logger.post_call_failure_hook.await_args.kwargs["user_api_key_dict"] == auth + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prefix,suffix", (("", ""), ("/gateway", "/"))) +async def test_legacy_sse_mount_emits_message_endpoint(prefix: str, suffix: str) -> None: + from starlette.applications import Starlette + from starlette.routing import Mount + from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing + from litellm.proxy._experimental.mcp_server import server as mcp_server + + app: Final = Starlette(routes=[Mount("/mcp", app=mcp_server.app)]) + incoming: Final[asyncio.Queue[Message]] = asyncio.Queue() + outgoing: Final[asyncio.Queue[Message]] = asyncio.Queue() + await incoming.put({"type": "http.request", "body": b"", "more_body": False}) + path: Final = f"{prefix}/mcp/sse{suffix}" + scope: Final[Scope] = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "GET", + "scheme": "http", + "path": path, + "raw_path": path.encode(), + "query_string": b"", + "root_path": prefix, + "server": ("localhost", 80), + "client": ("127.0.0.1", 1234), + "headers": [(b"accept", b"text/event-stream")], + } + auth: Final = UserAPIKeyAuth(api_key="test-owner") + with ( + patch.object( + mcp_server, "extract_mcp_auth_context", AsyncMock(return_value=(auth, None, None, None, None, None)) + ), + patch.object(mcp_server, "_raise_preemptive_401_for_unauthenticated_servers", AsyncMock()), + patch.object(mcp_server, "_check_passthrough_upstream_auth", AsyncMock()), + patch.object(mcp_server.operations, "_raise_if_initialize_grants_no_mcp_servers", AsyncMock()), + patch.object(mcp_server, "_SESSION_MANAGERS_INITIALIZED", True), + ): + task: Final = asyncio.create_task(app(scope, incoming.get, outgoing.put)) + try: + start: Final = await asyncio.wait_for(outgoing.get(), 2) + assert start["type"] == "http.response.start" + assert start["status"] == 200 + endpoint_frame: Final = await asyncio.wait_for(outgoing.get(), 2) + frame: Final = endpoint_frame["body"].decode() + assert "event: endpoint" in frame + endpoint: Final = frame.split("data: ", 1)[1].splitlines()[0] + assert endpoint.startswith(f"{prefix}/mcp/sse/messages?session_id=") + message_path, query = endpoint.split("?", 1) + + async def post(body: bytes) -> int: + messages: Final[asyncio.Queue[Message]] = asyncio.Queue() + requests: Final[asyncio.Queue[Message]] = asyncio.Queue() + await requests.put({"type": "http.request", "body": body, "more_body": False}) + post_scope: Final[Scope] = { + **scope, + "method": "POST", + "path": message_path + suffix, + "raw_path": (message_path + suffix).encode(), + "query_string": query.encode(), + "root_path": prefix, + "headers": [(b"content-type", b"application/json")], + } + await asyncio.wait_for(app(post_scope, requests.get, messages.put), 2) + return (await messages.get())["status"] + + initialization: Final = json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": LATEST_HANDSHAKE_VERSION, + "capabilities": {}, + "clientInfo": {"name": "legacy-client", "version": "1"}, + }, + } + ).encode() + assert await post(initialization) == 202 + reply: Final = (await asyncio.wait_for(outgoing.get(), 2))["body"].decode() + initialized: Final = json.loads(reply.split("data: ", 1)[1].splitlines()[0]) + assert initialized["id"] == 1 + assert initialized["result"]["serverInfo"]["name"] == "litellm-mcp-server" + + assert await post(b'{"jsonrpc":"2.0","method":"notifications/initialized"}') == 202 + for request_id, marker in ((2, "first-post"), (3, "second-post")): + post_auth: Final = UserAPIKeyAuth(api_key="test-owner", user_id=marker) + listing: Final = AsyncMock(return_value=AggregateToolListing(tools=[], outcomes={})) + with ( + patch.object( + mcp_server, + "extract_mcp_auth_context", + AsyncMock(return_value=(post_auth, None, [marker], {marker: {"Authorization": marker}}, {"Authorization": marker}, {"x-request-marker": marker})), + ), + patch.object(mcp_server.operations, "_get_tools_from_mcp_servers", listing), + ): + assert ( + await post(json.dumps({"jsonrpc": "2.0", "id": request_id, "method": "tools/list"}).encode()) + == 202 + ) + listed_frame: Final = (await asyncio.wait_for(outgoing.get(), 2))["body"].decode() + listed: Final = json.loads(listed_frame.split("data: ", 1)[1].splitlines()[0]) + assert listed["id"] == request_id + assert listed["result"]["tools"] == [] + listing.assert_awaited_once() + assert listing.await_args.kwargs["user_api_key_auth"].user_id == marker + assert listing.await_args.kwargs["mcp_servers"] == [marker] + assert listing.await_args.kwargs["mcp_server_auth_headers"] == {marker: {"Authorization": marker}} + assert listing.await_args.kwargs["oauth2_headers"] == {"Authorization": marker} + assert listing.await_args.kwargs["raw_headers"] == {"x-request-marker": marker} + + stranger: Final = UserAPIKeyAuth(api_key="different-owner") + with patch.object( + mcp_server, "extract_mcp_auth_context", AsyncMock(return_value=(stranger, None, None, None, None, None)) + ): + assert await post(initialization) == 404 + finally: + await incoming.put({"type": "http.disconnect"}) + await asyncio.wait_for(task, 2) + assert await post(initialization) == 404 diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 9f42a523350..7725aca1948 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -14470,6 +14470,21 @@ async def test_request_selected_during_guardrail_runs_concurrently_with_tool(mon assert result.content[0].text == "executed" +@pytest.mark.asyncio +@pytest.mark.parametrize("in_config,in_db,expected", [(True, False, True), (False, True, False), (True, True, False)]) +async def test_server_response_identifies_read_only_config(in_config, in_db, expected): + manager = MCPServerManager() + server = MCPServer(server_id="source-server", name="source_server", transport=MCPTransport.http) + manager.config_mcp_servers = {server.server_id: server} if in_config else {} + manager.registry = {server.server_id: server} if in_db else {} + + listed = await manager.get_all_mcp_servers_unfiltered() + + assert len(listed) == 1 + assert listed[0].model_dump().get("is_config") is expected + assert manager._build_mcp_server_table(server).model_dump().get("is_config") is expected + + @pytest.mark.asyncio @pytest.mark.parametrize("with_caller,legacy_factory", [(True, False), (False, False), (True, True)]) async def test_client_sampling_does_not_fill_explicit_context_from_another_ambient_caller(with_caller, legacy_factory): diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_utils.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_utils.py index 842859e5a1e..8d10219796b 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_utils.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_utils.py @@ -4,7 +4,7 @@ import pytest from fastapi import HTTPException from litellm.proxy._experimental.mcp_server.utils import ( - _upstream_credential_headers, + upstream_credential_headers, build_synthetic_mcp_request, logging_safe_mcp_headers, validate_and_normalize_mcp_server_payload, @@ -189,8 +189,8 @@ class TestLoggingSafeMcpHeaders: """clean_headers already strips authorization, and claiming it here would change which header authenticated_with_header resolves to on a config that lists it by design.""" with _configured_servers(_server_forwarding("Authorization", "X-GitHub-Token")): - assert "authorization" not in _upstream_credential_headers(["authorization", "x-github-token"]) - assert "x-github-token" in _upstream_credential_headers(["authorization", "x-github-token"]) + assert "authorization" not in upstream_credential_headers(["authorization", "x-github-token"]) + assert "x-github-token" in upstream_credential_headers(["authorization", "x-github-token"]) def test_keeps_headers_when_no_server_forwards_them(self): with _configured_servers(_server_forwarding("x-github-token")): diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py index 9a9ccd9a213..801f61aa498 100644 --- a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py @@ -198,6 +198,62 @@ class TestProxyExceptionAnthropicEnvelope: assert fallback.status_code == 500 assert json.loads(fallback.body)["error"]["type"] == "api_error" + @staticmethod + def _call_id_error_response(general_settings, provider_specific_fields=None): + import litellm.proxy.anthropic_endpoints.endpoints as ep + from litellm.proxy._types import ProxyException + + request = MagicMock() + request.headers = {} + exc = ProxyException( + message="Rate limit exceeded", + type="rate_limit_error", + param=None, + code=429, + headers={"x-litellm-call-id": "call-8302"}, + provider_specific_fields=provider_specific_fields, + ) + with patch("litellm.proxy.proxy_server.general_settings", general_settings): + return ep._anthropic_error_json_response(exc, request) + + def test_anthropic_error_copies_the_call_id_into_the_error_when_opted_in(self): + """With include_call_id_in_error_body on, error.litellm_call_id is byte-identical to + the x-litellm-call-id header and lives inside the error object, which is what the + Anthropic SDK keeps as e.body.""" + response = self._call_id_error_response({"include_call_id_in_error_body": True}) + + assert response.headers["x-litellm-call-id"] == "call-8302" + assert json.loads(response.body) == { + "type": "error", + "error": { + "type": "rate_limit_error", + "message": "Rate limit exceeded", + "litellm_call_id": "call-8302", + }, + } + + def test_anthropic_error_keeps_provider_specific_fields_next_to_the_call_id(self): + response = self._call_id_error_response( + {"include_call_id_in_error_body": True}, + provider_specific_fields={"guardrail": "keyword-block"}, + ) + + assert json.loads(response.body)["error"] == { + "type": "rate_limit_error", + "message": "Rate limit exceeded", + "provider_specific_fields": {"guardrail": "keyword-block"}, + "litellm_call_id": "call-8302", + } + + def test_anthropic_error_leaves_the_envelope_alone_when_opted_out(self): + response = self._call_id_error_response({}) + + assert response.headers["x-litellm-call-id"] == "call-8302" + assert json.loads(response.body) == { + "type": "error", + "error": {"type": "rate_limit_error", "message": "Rate limit exceeded"}, + } + class TestHttpExceptionDictDetail: @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 0cab6535f3d..ffba53a5b7a 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -75,6 +75,8 @@ from litellm.constants import ( ) from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper +from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler +from prisma.errors import DataError from litellm.proxy.common_utils.user_api_key_cache import ( END_USER_RESTRICTED_REGISTRY_OVERFLOW_SENTINEL, TAG_REGISTRY_OVERFLOW_SENTINEL, @@ -892,36 +894,80 @@ async def test_get_user_object_upsert_sets_budget_reset_at(monkeypatch, has_budg assert "budget_reset_at" not in creation_args -@pytest.mark.asyncio -async def test_get_user_object_wraps_db_outage_as_valueerror_preserving_context(): - """Pin get_user_object's exception contract: it catches every DB failure in a broad except and - re-raises a bare ValueError, so a real outage survives only as __context__ rather than as the - exception type. The MCP dcr_bridge admission and refresh paths depend on this to tell a transient - outage (retry, 503) from a missing user (fail closed), which is why they classify across the cause - chain instead of the top exception's type. If this wrapping ever changes, that classification must - change with it, so this test guards the contract the callers rely on.""" - from unittest.mock import AsyncMock, MagicMock, patch +def _user_read_raising(error: Exception) -> tuple[MagicMock, MagicMock]: + prisma_client = MagicMock() + prisma_client.db.litellm_usertable.find_unique = AsyncMock(side_effect=error) + cache = MagicMock() + cache.async_get_cache = AsyncMock(return_value=None) + cache.async_set_cache = AsyncMock() + return prisma_client, cache - mock_prisma_client = MagicMock() - mock_prisma_client.db = AsyncMock() - mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( - side_effect=ConnectionError("can't reach database server") - ) - mock_cache = MagicMock() - mock_cache.async_get_cache = AsyncMock(return_value=None) - mock_cache.async_set_cache = AsyncMock() + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "outage", + [ + httpx.ConnectError("All connection attempts failed"), + httpx.ReadTimeout("timed out"), + DataError( + data={ + "user_facing_error": { + "message": "Can't reach database server at `127.0.0.1:41071`", + "error_code": "P1001", + } + } + ), + ], + ids=["connect_error", "read_timeout", "p1001_as_data_error"], +) +async def test_get_user_object_surfaces_a_db_outage_as_503_not_as_a_missing_user(outage): + from litellm.proxy.auth.auth_exception_handler import _as_proxy_exception + + prisma_client, cache = _user_read_raising(outage) with patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True): - with pytest.raises(ValueError, match="User doesn't exist in db\\.") as exc_info: + with pytest.raises(type(outage)) as raised: await get_user_object( - user_id="outage-contract-probe-user", - prisma_client=mock_prisma_client, - user_api_key_cache=mock_cache, + user_id="outage-probe-user", + prisma_client=prisma_client, + user_api_key_cache=cache, user_id_upsert=False, proxy_logging_obj=None, ) - assert isinstance(exc_info.value.__context__, ConnectionError) + assert raised.value is outage + assert PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(raised.value) is outage + surfaced = _as_proxy_exception(raised.value) + assert (surfaced.code, surfaced.type) == ("503", ProxyErrorTypes.no_db_connection) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "failure", + [ + DataError(data={"user_facing_error": {"message": "invalid byte sequence for encoding UTF8: 0x00"}}), + RuntimeError("row validation failed"), + ], + ids=["query_level_data_error", "runtime_error"], +) +async def test_get_user_object_still_reports_a_non_outage_read_failure_as_a_missing_user(failure): + from litellm.proxy.auth.auth_exception_handler import _as_proxy_exception + + prisma_client, cache = _user_read_raising(failure) + + with patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True): + with pytest.raises(ValueError, match="User doesn't exist in db\\.") as raised: + await get_user_object( + user_id="data-error-probe-user", + prisma_client=prisma_client, + user_api_key_cache=cache, + user_id_upsert=False, + proxy_logging_obj=None, + ) + + assert raised.value.__context__ is failure + surfaced = _as_proxy_exception(raised.value) + assert (surfaced.code, surfaced.type) == ("401", ProxyErrorTypes.auth_error) @pytest.mark.asyncio @@ -8011,7 +8057,6 @@ def test_model_has_no_cost_mapping_no_model_or_router_is_false(): [ "azure/speech/azure-tts", "mistral/mistral-ocr-latest", - "vertex_ai/imagen-3.0-generate-001", "dashscope/qwen-flash", ], ) diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index 965acd57bf3..83ac56c4c85 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -2727,6 +2727,30 @@ class TestIsRequestBodySafeBlocksBedrockProjectOverride: ) +class TestIsRequestBodySafeBlocksClaudePlatformWorkspaceOverride: + @pytest.mark.parametrize( + "alias", ["workspace_id", "aws_workspace_id", "anthropic_workspace_id", "anthropic-workspace-id"] + ) + def test_workspace_alias_in_request_body_is_rejected(self, alias): + with pytest.raises(ValueError, match=alias): + is_request_body_safe( + request_body={"model": "gpt-4", alias: "wrkspc_attacker"}, + general_settings={}, + llm_router=None, + model="gpt-4", + ) + + def test_admin_opt_in_proxy_wide_allows_workspace_id(self): + assert ( + is_request_body_safe( + request_body={"model": "gpt-4", "workspace_id": "wrkspc_byok"}, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="gpt-4", + ) + is True + ) + class TestIsRequestBodySafeBlocksRustOptIn: """``rust`` hands the whole call to the Rust core, which signs and sends with its own HTTP client rather than the one the deployment configured, and diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index 0969a913605..f8b9043a23f 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -28,10 +28,12 @@ from litellm.proxy._types import ( ) from litellm.caching.dual_cache import DualCache from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry +from litellm.proxy.auth.auth_checks import TeamNotFoundError from litellm.proxy.auth.handle_jwt import ( JWKS_FETCH_ATTEMPTS, STALE_CACHE_KEY_PREFIX, STALE_WRITTEN_AT_CACHE_KEY_PREFIX, + HeaderTeam, JWKSUnreachableError, JWTAuthManager, JWTHandler, @@ -1993,29 +1995,41 @@ async def test_auth_builder_oidc_enabled_falls_back_to_jwt_auth_for_jwt_tokens() assert result["user_object"] == user_object -def test_get_team_id_from_header(): - """Test get_team_id_from_header returns team when valid, None when missing, raises on invalid.""" - from fastapi import HTTPException - - # Valid team in allowed list - result = JWTAuthManager.get_team_id_from_header( +@pytest.mark.asyncio +async def test_resolve_team_from_header_returns_allowed_id_none_without_header_and_403_on_invalid(): + """Without a DB, x-litellm-team-id resolves to the team when it names an allowed + team id, to None when the header is absent, and to a 403 for any other value.""" + allowed = await JWTAuthManager.resolve_team_from_header( request_headers={"x-litellm-team-id": "team-1"}, allowed_team_ids={"team-1", "team-2"}, + fallback_to_db_teams=False, + prisma_client=None, + user_api_key_cache=MagicMock(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), ) - assert result == "team-1" + assert allowed == HeaderTeam(header_value="team-1", team_id="team-1") - # No header returns None - result = JWTAuthManager.get_team_id_from_header( + absent = await JWTAuthManager.resolve_team_from_header( request_headers={"authorization": "Bearer token"}, allowed_team_ids={"team-1"}, + fallback_to_db_teams=False, + prisma_client=None, + user_api_key_cache=MagicMock(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), ) - assert result is None + assert absent is None - # Invalid team raises 403 with pytest.raises(HTTPException) as exc_info: - JWTAuthManager.get_team_id_from_header( + await JWTAuthManager.resolve_team_from_header( request_headers={"x-litellm-team-id": "invalid-team"}, allowed_team_ids={"team-1", "team-2"}, + fallback_to_db_teams=False, + prisma_client=None, + user_api_key_cache=MagicMock(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), ) assert exc_info.value.status_code == 403 @@ -3350,15 +3364,26 @@ async def test_auth_builder_single_team_db_fallback_when_jwt_has_no_team( mock_get_membership.assert_not_called() -@pytest.mark.asyncio -async def test_auth_builder_single_team_fallback_membership_error_skips_no_raise(): - """ - get_team_object succeeds but get_team_membership raises — do not set team; no exception. - """ - from fastapi import HTTPException +class _UnreachableMembershipPrisma: + class db: + class litellm_teammembership: + @staticmethod + async def find_unique(where: dict[str, dict[str, str]], include: dict[str, bool]) -> None: + raise httpx.ConnectError("All connection attempts failed") - user_id = "u_mem_fail" - team_id_val = "team_mem_fail" + +@pytest.mark.asyncio +async def test_auth_builder_single_team_fallback_membership_outage_raises_instead_of_dropping_the_team(): + """ + get_team_object succeeds but the membership read hits a database outage: the + outage propagates (auth maps it to 503) instead of the team being dropped. + """ + from litellm.proxy.auth.auth_exception_handler import _as_proxy_exception + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.utils import ProxyLogging + + user_id = "u_mem_outage" + team_id_val = "team_mem_outage" user_object = LiteLLM_UserTable( user_id=user_id, user_role=LitellmUserRoles.INTERNAL_USER, @@ -3367,6 +3392,7 @@ async def test_auth_builder_single_team_fallback_membership_error_skips_no_raise team_table = LiteLLM_TeamTable(team_id=team_id_val) jwt_handler = JWTHandler() jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth() + cache = UserApiKeyCache() with ( patch.object(jwt_handler, "auth_jwt", new_callable=AsyncMock) as mock_auth_jwt, @@ -3416,34 +3442,26 @@ async def test_auth_builder_single_team_fallback_membership_error_skips_no_raise "litellm.proxy.auth.handle_jwt.get_team_object", new_callable=AsyncMock, ) as mock_get_team, - patch( - "litellm.proxy.auth.handle_jwt.get_team_membership", - new_callable=AsyncMock, - ) as mock_get_membership, ): mock_auth_jwt.return_value = {"sub": user_id, "scope": ""} mock_get_team.return_value = team_table - mock_get_membership.side_effect = HTTPException( - status_code=500, detail="membership lookup failed" - ) - result = await JWTAuthManager.auth_builder( - api_key="test_jwt_token", - jwt_handler=jwt_handler, - request_data={"model": "gpt-4"}, - general_settings={"enforce_rbac": False}, - route="/chat/completions", - prisma_client=None, - user_api_key_cache=None, - parent_otel_span=None, - proxy_logging_obj=None, - ) + with pytest.raises(httpx.ConnectError) as raised: + await JWTAuthManager.auth_builder( + api_key="test_jwt_token", + jwt_handler=jwt_handler, + request_data={"model": "gpt-4"}, + general_settings={"enforce_rbac": False}, + route="/chat/completions", + prisma_client=_UnreachableMembershipPrisma(), + user_api_key_cache=cache, + parent_otel_span=None, + proxy_logging_obj=ProxyLogging(user_api_key_cache=cache), + ) - assert result["team_id"] is None - assert result["team_object"] is None - assert result["team_membership"] is None - mock_get_team.assert_called() - mock_get_membership.assert_called_once() + mock_get_team.assert_called() + surfaced = _as_proxy_exception(raised.value) + assert (surfaced.code, surfaced.type) == ("503", ProxyErrorTypes.no_db_connection) # --------------------------------------------------------------------------- @@ -5305,32 +5323,23 @@ def test_build_decode_kwargs_warns_for_unscoped_global_fallback_in_mixed_deploym # --------------------------------------------------------------------------- -def test_get_team_id_from_header_defers_to_db_membership_only_without_jwt_claims(): - """With fallback_to_db_teams=True, an x-litellm-team-id header is accepted - provisionally only when the JWT carries no team claims (allowed set empty). - When the JWT does carry team claims, the header must still be validated +@pytest.mark.asyncio +async def test_resolve_team_from_header_defers_to_db_membership_only_without_jwt_claims(): + """With fallback_to_db_teams=True, an x-litellm-team-id header naming an existing + team is accepted provisionally only when the JWT carries no team claims (allowed + set empty). When the JWT does carry team claims, the header must still be validated against them, and the flag-off behavior must keep rejecting unknown teams.""" - deferred = JWTAuthManager.get_team_id_from_header( - request_headers={"x-litellm-team-id": "team-from-db"}, - allowed_team_ids=set(), - fallback_to_db_teams=True, - ) - assert deferred == "team-from-db" + known_ids = frozenset({"team-from-db"}) + + deferred, _, _ = await _resolve_header("team-from-db", set(), True, _teams_by_id(known_ids), _team_alias_lookup_404) + assert deferred == HeaderTeam(header_value="team-from-db", team_id="team-from-db") with pytest.raises(HTTPException) as exc_info: - JWTAuthManager.get_team_id_from_header( - request_headers={"x-litellm-team-id": "team-x"}, - allowed_team_ids={"team-1", "team-2"}, - fallback_to_db_teams=True, - ) + await _resolve_header("team-x", {"team-1", "team-2"}, True, _teams_by_id(known_ids), _team_alias_lookup_404) assert exc_info.value.status_code == 403 with pytest.raises(HTTPException): - JWTAuthManager.get_team_id_from_header( - request_headers={"x-litellm-team-id": "team-from-db"}, - allowed_team_ids=set(), - fallback_to_db_teams=False, - ) + await _resolve_header("team-from-db", set(), False, _teams_by_id(known_ids), _team_alias_lookup_404) @pytest.mark.asyncio @@ -5784,6 +5793,7 @@ def test_validate_header_team_in_db_membership_does_not_leak_team_ids(): JWTAuthManager._validate_header_team_in_db_membership( team_id="outsider_team", user_object=user_object, + header_value="outsider_team", ) detail = exc_info.value.detail @@ -5793,6 +5803,43 @@ def test_validate_header_team_in_db_membership_does_not_leak_team_ids(): assert "outsider_team" in detail +async def _team_lookup_404(team_id, **kwargs): + raise HTTPException( + status_code=404, + detail=f"Team doesn't exist in db. Team={team_id}. Create team via `/team/new` call.", + ) + + +async def _team_alias_lookup_404(team_alias, **kwargs): + raise HTTPException( + status_code=404, + detail={"error": f"Team with alias '{team_alias}' doesn't exist in db. Create team via `/team/new` call."}, + ) + + +def _teams_by_alias(aliases: Mapping[str, str]): + """An alias lookup over `aliases` (alias -> team_id) that 404s like the real one otherwise.""" + + async def lookup(team_alias, **kwargs): + if team_alias not in aliases: + return await _team_alias_lookup_404(team_alias) + return LiteLLM_TeamTable(team_id=aliases[team_alias], team_alias=team_alias) + + return lookup + + +def _teams_by_id(team_ids: frozenset[str]): + """A team lookup that knows exactly `team_ids` and, like the real one, reports + any other id as provably absent.""" + + async def lookup(team_id, **kwargs): + if team_id not in team_ids: + raise TeamNotFoundError(team_id=team_id) + return LiteLLM_TeamTable(team_id=team_id) + + return lookup + + async def _run_auth_builder_with_header_team( jwt_auth_config: LiteLLM_JWTAuth, token: dict, @@ -5800,6 +5847,8 @@ async def _run_auth_builder_with_header_team( user_object: LiteLLM_UserTable, fake_get_team, allowed_team_ids: set, + fake_get_team_by_alias=_team_alias_lookup_404, + route: str = "/chat/completions", ): jwt_handler = JWTHandler() jwt_handler.litellm_jwtauth = jwt_auth_config @@ -5844,14 +5893,19 @@ async def _run_auth_builder_with_header_team( new_callable=AsyncMock, side_effect=fake_get_team, ), + patch( + "litellm.proxy.auth.handle_jwt.get_team_object_by_alias", + new_callable=AsyncMock, + side_effect=fake_get_team_by_alias, + ), ): return await JWTAuthManager.auth_builder( api_key="test_jwt_token", jwt_handler=jwt_handler, request_data={"model": "gpt-4"}, general_settings={"enforce_rbac": False}, - route="/chat/completions", - prisma_client=None, + route=route, + prisma_client=MagicMock(), user_api_key_cache=None, parent_otel_span=None, proxy_logging_obj=None, @@ -5859,13 +5913,6 @@ async def _run_auth_builder_with_header_team( ) -async def _team_lookup_404(team_id, **kwargs): - raise HTTPException( - status_code=404, - detail=f"Team doesn't exist in db. Team={team_id}. Create team via `/team/new` call.", - ) - - @pytest.mark.asyncio async def test_auth_builder_header_team_not_found_matches_non_membership_denial() -> ( None @@ -6938,6 +6985,304 @@ async def test_auth_builder_header_team_enforces_team_allowed_routes_under_db_fa assert result["team_id"] == header_team +async def _resolve_header( + header_value: str, + allowed_team_ids: set[str], + fallback_to_db_teams: bool, + fake_get_team, + fake_get_team_by_alias, +) -> tuple[HeaderTeam | None, AsyncMock, AsyncMock]: + with ( + patch( + "litellm.proxy.auth.handle_jwt.get_team_object", + new_callable=AsyncMock, + side_effect=fake_get_team, + ) as by_id, + patch( + "litellm.proxy.auth.handle_jwt.get_team_object_by_alias", + new_callable=AsyncMock, + side_effect=fake_get_team_by_alias, + ) as by_alias, + ): + resolved = await JWTAuthManager.resolve_team_from_header( + request_headers={"X-LiteLLM-Team-Id": header_value}, + allowed_team_ids=allowed_team_ids, + fallback_to_db_teams=fallback_to_db_teams, + prisma_client=MagicMock(), + user_api_key_cache=MagicMock(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + return resolved, by_id, by_alias + + +@pytest.mark.asyncio +async def test_resolve_team_from_header_accepts_the_alias_of_an_allowed_team(): + """x-litellm-team-id may carry the team alias instead of the team id (LIT-7181). + The alias resolves to its team id before the allowed-teams check, so a + caller whose JWT grants team_a gets team_a whether it sends the id or the + alias, and the id path never pays for an alias lookup.""" + aliases = {"alias_a": "team_a", "alias_b": "team_b"} + + by_alias_value, lookups_by_id, lookups_by_alias = await _resolve_header( + "alias_a", {"team_a"}, False, _team_lookup_404, _teams_by_alias(aliases) + ) + assert by_alias_value == HeaderTeam(header_value="alias_a", team_id="team_a") + lookups_by_alias.assert_awaited_once() + assert lookups_by_alias.await_args.kwargs["team_alias"] == "alias_a" + lookups_by_id.assert_not_awaited() + + by_id_value, lookups_by_id, lookups_by_alias = await _resolve_header( + "team_a", {"team_a"}, False, _team_lookup_404, _teams_by_alias(aliases) + ) + assert by_id_value == HeaderTeam(header_value="team_a", team_id="team_a") + lookups_by_alias.assert_not_awaited() + lookups_by_id.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_resolve_team_from_header_denies_aliases_of_teams_the_jwt_does_not_grant(): + """An alias that exists but names a team outside the JWT's allowed teams is + refused with the same 403 as an unknown value, and the detail names only + what the caller sent, so the response reveals neither that the alias exists + nor which team id it maps to. Because the id and the alias lookups both ran + before the denial, the detail says the value resolved to neither form and + lists the team ids the JWT does allow.""" + aliases = {"alias_a": "team_a", "alias_b": "team_b"} + + with pytest.raises(HTTPException) as other_team: + await _resolve_header("alias_b", {"team_a"}, False, _team_lookup_404, _teams_by_alias(aliases)) + with pytest.raises(HTTPException) as unknown: + await _resolve_header("no_such", {"team_a"}, False, _team_lookup_404, _teams_by_alias(aliases)) + + assert other_team.value.status_code == 403 + assert unknown.value.status_code == 403 + assert "team_b" not in other_team.value.detail + assert other_team.value.detail.replace("alias_b", "") == unknown.value.detail.replace("no_such", "") + assert unknown.value.detail == ( + "x-litellm-team-id 'no_such' does not resolve to a team id or a unique team alias in your JWT's allowed " + "teams. Allowed team ids: ['team_a']" + ) + + +@pytest.mark.asyncio +async def test_resolve_team_from_header_under_db_fallback_tries_the_id_before_the_alias(): + """Under fallback_to_db_teams a claimless JWT's header is provisional: a value + that is an existing team id resolves to itself without an alias lookup, a + value that is only an alias resolves to that team's id, and a value that is + neither gets the membership denial the id path already uses.""" + known_ids = frozenset({"team_a"}) + aliases = {"alias_a": "team_a"} + + as_id, _, lookups_by_alias = await _resolve_header( + "team_a", set(), True, _teams_by_id(known_ids), _teams_by_alias(aliases) + ) + assert as_id == HeaderTeam(header_value="team_a", team_id="team_a") + lookups_by_alias.assert_not_awaited() + + as_alias, _, _ = await _resolve_header("alias_a", set(), True, _teams_by_id(known_ids), _teams_by_alias(aliases)) + assert as_alias == HeaderTeam(header_value="alias_a", team_id="team_a") + + with pytest.raises(HTTPException) as neither: + await _resolve_header("ghost", set(), True, _teams_by_id(known_ids), _teams_by_alias(aliases)) + assert neither.value.status_code == 403 + assert neither.value.detail == ( + "x-litellm-team-id 'ghost' does not resolve to a team id or a unique team alias among your team memberships." + ) + + +@pytest.mark.asyncio +async def test_resolve_team_from_header_under_db_fallback_never_aliases_a_team_id_it_could_not_read(): + """Only a team row the database provably lacks falls through to the alias + lookup. When the id read fails for any other reason (the generic 404 the + team lookup uses for an unreadable database) the value keeps the id path's + membership denial, so an outage can never turn a team id into the team + that happens to carry it as an alias.""" + aliases = {"team_a": "team_b"} + + with ( + patch("litellm.proxy.auth.handle_jwt.get_team_object", new_callable=AsyncMock, side_effect=_team_lookup_404), + patch( + "litellm.proxy.auth.handle_jwt.get_team_object_by_alias", + new_callable=AsyncMock, + side_effect=_teams_by_alias(aliases), + ) as lookups_by_alias, + pytest.raises(HTTPException) as unreadable, + ): + await JWTAuthManager.resolve_team_from_header( + request_headers={"x-litellm-team-id": "team_a"}, + allowed_team_ids=set(), + fallback_to_db_teams=True, + prisma_client=MagicMock(), + user_api_key_cache=MagicMock(), + parent_otel_span=None, + proxy_logging_obj=MagicMock(), + ) + + assert unreadable.value.status_code == 403 + assert unreadable.value.detail == ( + "x-litellm-team-id 'team_a' does not resolve to a team id or a unique team alias among your team memberships." + ) + lookups_by_alias.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_resolve_team_from_header_denies_a_duplicate_alias_like_an_unknown_value_but_surfaces_lookup_errors(): + """An alias two teams share resolves to no single team, so it is denied with + the very 403 an unknown value gets, under claims and under the DB fallback + alike: the caller cannot be checked against either team, so telling it the + alias is shared would let any JWT probe which aliases exist. The detail says + the value resolved to no team id or unique alias, which is true for both. + A lookup failure (5xx) is not disguised as a denial and propagates as is.""" + + async def duplicate_alias(team_alias, **kwargs): + raise HTTPException(status_code=400, detail={"error": f"Multiple teams found with alias '{team_alias}'."}) + + async def db_down(team_alias, **kwargs): + raise HTTPException(status_code=500, detail={"error": f"Error looking up team by alias '{team_alias}'"}) + + with pytest.raises(HTTPException) as duplicate: + await _resolve_header("shared_alias", {"team_a"}, False, _team_lookup_404, duplicate_alias) + with pytest.raises(HTTPException) as unknown: + await _resolve_header("shared_alias", {"team_a"}, False, _team_lookup_404, _team_alias_lookup_404) + assert duplicate.value.status_code == 403 + assert duplicate.value.detail == unknown.value.detail + assert duplicate.value.detail == ( + "x-litellm-team-id 'shared_alias' does not resolve to a team id or a unique team alias in your JWT's " + "allowed teams. Allowed team ids: ['team_a']" + ) + + known_ids = frozenset({"team_a"}) + with pytest.raises(HTTPException) as duplicate_under_fallback: + await _resolve_header("shared_alias", set(), True, _teams_by_id(known_ids), duplicate_alias) + with pytest.raises(HTTPException) as unknown_under_fallback: + await _resolve_header("shared_alias", set(), True, _teams_by_id(known_ids), _team_alias_lookup_404) + assert duplicate_under_fallback.value.status_code == 403 + assert duplicate_under_fallback.value.detail == unknown_under_fallback.value.detail + assert duplicate_under_fallback.value.detail == ( + "x-litellm-team-id 'shared_alias' does not resolve to a team id or a unique team alias among your team " + "memberships." + ) + + with pytest.raises(HTTPException) as failure: + await _resolve_header("alias_a", {"team_a"}, False, _team_lookup_404, db_down) + assert failure.value.status_code == 500 + + +@pytest.mark.asyncio +async def test_auth_builder_header_alias_binds_the_aliased_team_under_claims_and_db_fallback(): + """End to end through auth_builder, x-litellm-team-id carrying a team alias + binds the request to the aliased team (result team_id is the canonical id) + both when the JWT grants that team by claim and when a claimless JWT relies + on fallback_to_db_teams and DB membership.""" + aliases = {"alias_member": "team_member"} + user_object = LiteLLM_UserTable( + user_id="u_alias", + user_role=LitellmUserRoles.INTERNAL_USER, + teams=["team_member"], + ) + + claims_config = LiteLLM_JWTAuth(team_ids_jwt_field="team_ids") + by_claim = await _run_auth_builder_with_header_team( + claims_config, + {"sub": "u_alias", "scope": "", "team_ids": ["team_member"]}, + "alias_member", + user_object, + _teams_by_id(frozenset({"team_member"})), + {"team_member"}, + _teams_by_alias(aliases), + ) + assert by_claim["team_id"] == "team_member" + assert by_claim["team_object"].team_id == "team_member" + + fallback_config = LiteLLM_JWTAuth(fallback_to_db_teams=True) + by_membership = await _run_auth_builder_with_header_team( + fallback_config, + {"sub": "u_alias", "scope": ""}, + "alias_member", + user_object, + _teams_by_id(frozenset({"team_member"})), + set(), + _teams_by_alias(aliases), + ) + assert by_membership["team_id"] == "team_member" + assert by_membership["team_object"].team_id == "team_member" + + +@pytest.mark.asyncio +async def test_auth_builder_header_alias_of_a_non_member_team_is_denied_like_an_unknown_value_under_db_fallback(): + """Under fallback_to_db_teams, an alias naming a team the user is not a member + of is denied with the exact same 403 as an unknown value, naming the alias + the caller sent rather than the team id it resolved to.""" + aliases = {"alias_other": "team_other"} + known_ids = frozenset({"team_member", "team_other"}) + user_object = LiteLLM_UserTable( + user_id="u_alias_outsider", + user_role=LitellmUserRoles.INTERNAL_USER, + teams=["team_member"], + ) + config = LiteLLM_JWTAuth(fallback_to_db_teams=True) + token = {"sub": "u_alias_outsider", "scope": ""} + + with pytest.raises(HTTPException) as outsider_alias: + await _run_auth_builder_with_header_team( + config, token, "alias_other", user_object, _teams_by_id(known_ids), set(), _teams_by_alias(aliases) + ) + with pytest.raises(HTTPException) as unknown: + await _run_auth_builder_with_header_team( + config, token, "alias_ghost", user_object, _teams_by_id(known_ids), set(), _teams_by_alias(aliases) + ) + + assert outsider_alias.value.status_code == 403 + assert unknown.value.status_code == 403 + assert "team_other" not in outsider_alias.value.detail + assert outsider_alias.value.detail.replace("alias_other", "") == unknown.value.detail.replace( + "alias_ghost", "" + ) + + +@pytest.mark.asyncio +async def test_auth_builder_header_alias_under_db_fallback_keeps_the_team_allowed_routes_gate(): + """Under fallback_to_db_teams, a member team selected by alias is still held + to team_allowed_routes, and the denial names the alias the caller sent.""" + aliases = {"alias_member": "team_member"} + user_object = LiteLLM_UserTable( + user_id="u_alias_routes", + user_role=LitellmUserRoles.INTERNAL_USER, + teams=["team_member"], + ) + config = LiteLLM_JWTAuth(fallback_to_db_teams=True, team_allowed_routes=["openai_routes"]) + token = {"sub": "u_alias_routes", "scope": ""} + + with pytest.raises(HTTPException) as exc_info: + await _run_auth_builder_with_header_team( + config, + token, + "alias_member", + user_object, + _teams_by_id(frozenset({"team_member"})), + set(), + _teams_by_alias(aliases), + route="/key/info", + ) + assert exc_info.value.status_code == 403 + assert exc_info.value.detail == ( + "Team 'alias_member' (from x-litellm-team-id header) is not allowed to access route '/key/info'." + ) + + allowed = await _run_auth_builder_with_header_team( + config, + token, + "alias_member", + user_object, + _teams_by_id(frozenset({"team_member"})), + set(), + _teams_by_alias(aliases), + route="/chat/completions", + ) + assert allowed["team_id"] == "team_member" + + @pytest.mark.asyncio async def test_sync_user_role_and_teams_singular_claim_only_recognized_under_flag(): """Reading the singular team claim during sync is scoped to fallback_to_db_teams. diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 8da93ba341b..e5179387f82 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -653,6 +653,10 @@ def test_virtual_key_llm_api_routes_denies_spend_logs_v2(): "/mcp-rest/tools/call", "/mcp/tools/list", "/token", + "/mcp/sse", + "/mcp/sse/", + "/mcp/sse/messages", + "/mcp/sse/messages/", ], ) def test_mcp_inference_routes_classified_as_llm_api(route): @@ -4310,3 +4314,26 @@ def test_project_delete_route_stays_proxy_admin_only(): valid_token=valid_token, request_data={}, ) + + +@pytest.mark.parametrize("route", ("/mcp/sse", "/mcp/sse/", "/mcp/sse/messages", "/mcp/sse/messages/")) +@pytest.mark.parametrize("route_group", ("mcp_routes", "llm_api_routes", "openai_routes")) +def test_legacy_sse_respects_virtual_key_route_permissions(route: str, route_group: str) -> None: + token: Final = UserAPIKeyAuth( + user_id="sse-caller", user_role=LitellmUserRoles.INTERNAL_USER, allowed_routes=[route_group] + ) + request: Final = Request({"type": "http", "method": "POST" if "messages" in route else "GET", "path": route}) + if route_group == "openai_routes": + with pytest.raises(HTTPException) as caught: + RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=token, request=request) + assert caught.value.status_code == 403 + return + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=None, + _user_role=LitellmUserRoles.INTERNAL_USER, + route=route, + request=request, + valid_token=token, + request_data={}, + ) + assert RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=token, request=request) diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index da36071a5b4..f03abe8f124 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -59,6 +59,7 @@ from litellm.proxy.auth.user_api_key_auth import ( _user_api_key_auth_builder, get_api_key, user_api_key_auth, + user_api_key_auth_websocket_for_model, ) from litellm.proxy.spend_tracking.carried_budget_state import carried_budget_metadata @@ -9043,3 +9044,90 @@ async def test_router_settings_model_group_alias_authorizes_target_for_team(monk await authorize() assert (await request.json())["model"] == target assert get_client_requested_model(request) == "AgentX-LLM" + + +@pytest.mark.asyncio +async def test_reserve_budget_after_common_checks_hands_the_reservation_to_the_request_state(): + from fastapi import Request + + request = Request(scope={"type": "http"}) + user_api_key_auth_obj = UserAPIKeyAuth(token="test_token") + reservation = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + + with patch( + "litellm.proxy.spend_tracking.budget_reservation.reserve_budget_for_request", + new=AsyncMock(return_value=reservation), + ): + await _reserve_budget_after_common_checks( + user_api_key_auth_obj=user_api_key_auth_obj, + request_data={"model": "gpt-4o"}, + route="/v1/batches/batch_123/cancel", + llm_router=None, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + skip_budget_checks=False, + general_settings={}, + request=request, + ) + + assert user_api_key_auth_obj.budget_reservation is reservation + assert request.state.budget_reservation is reservation + assert request.scope["state"]["budget_reservation"] is reservation + + +@pytest.mark.asyncio +async def test_reserve_budget_after_common_checks_clears_the_request_state_when_budget_checks_skip(): + from fastapi import Request + + request = Request(scope={"type": "http", "state": {"budget_reservation": {"reserved_cost": 0.5}}}) + + await _reserve_budget_after_common_checks( + user_api_key_auth_obj=UserAPIKeyAuth(token="test_token"), + request_data={"model": "free-model"}, + route="/v1/chat/completions", + llm_router=None, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + skip_budget_checks=True, + general_settings={}, + request=request, + ) + + assert request.state.budget_reservation is None + + +@pytest.mark.asyncio +async def test_websocket_auth_hands_the_reservation_to_the_socket_state(): + from fastapi import WebSocket + + reservation = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + websocket = WebSocket( + scope={ + "type": "websocket", + "path": "/v1/realtime", + "headers": [(b"authorization", b"Bearer sk-1234")], + "query_string": b"model=gpt-realtime", + }, + receive=AsyncMock(), + send=AsyncMock(), + ) + + async def auth_that_reserves(request, api_key): + request.state.budget_reservation = reservation + return UserAPIKeyAuth(token="hashed", budget_reservation=reservation) + + with patch( + "litellm.proxy.auth.user_api_key_auth.user_api_key_auth", + new=AsyncMock(side_effect=auth_that_reserves), + ): + result = await user_api_key_auth_websocket_for_model(websocket, model="gpt-realtime") + + assert result.budget_reservation == reservation + assert websocket.state.budget_reservation is reservation + assert websocket.scope["state"]["budget_reservation"] is reservation diff --git a/tests/test_litellm/proxy/common_utils/test_callback_config_validation.py b/tests/test_litellm/proxy/common_utils/test_callback_config_validation.py index a707c92dc15..33ae986a511 100644 --- a/tests/test_litellm/proxy/common_utils/test_callback_config_validation.py +++ b/tests/test_litellm/proxy/common_utils/test_callback_config_validation.py @@ -3,6 +3,7 @@ import pytest from litellm.proxy.common_utils.callback_config_validation import ( callback_config_error, conflicting_span_scope_error, + cross_entry_family_error, logging_metadata_config_error, ) @@ -67,8 +68,16 @@ def test_one_span_scope_per_team(new_vars, stored, rejected): def test_key_logging_entries_may_not_disagree_on_the_span_scope(): disagreeing = { "logging": [ - {"callback_name": "langfuse_otel", "callback_type": "success", "callback_vars": {"langfuse_span_scope": "full"}}, - {"callback_name": "langfuse_otel", "callback_type": "failure", "callback_vars": {"langfuse_span_scope": "llm_only"}}, + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": {"langfuse_span_scope": "full"}, + }, + { + "callback_name": "langfuse_otel", + "callback_type": "failure", + "callback_vars": {"langfuse_span_scope": "llm_only"}, + }, ] } error = logging_metadata_config_error(disagreeing) @@ -76,9 +85,46 @@ def test_key_logging_entries_may_not_disagree_on_the_span_scope(): agreeing = { "logging": [ - {"callback_name": "langfuse_otel", "callback_type": "success", "callback_vars": {"langfuse_span_scope": "llm_only"}}, - {"callback_name": "langfuse_otel", "callback_type": "failure", "callback_vars": {"langfuse_span_scope": "llm_only"}}, + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": {"langfuse_span_scope": "llm_only"}, + }, + { + "callback_name": "langfuse_otel", + "callback_type": "failure", + "callback_vars": {"langfuse_span_scope": "llm_only"}, + }, {"callback_name": "otel", "callback_type": "success", "callback_vars": {}}, ] } assert logging_metadata_config_error(agreeing) is None + + +@pytest.mark.parametrize("var", ["arize_success_sampling_rate", "arize_error_sampling_rate"]) +@pytest.mark.parametrize("bad", ["1.5", "-0.1", "abc", "nan", "inf"]) +def test_callback_config_error_rejects_out_of_range_arize_sampling_rate(var, bad): + error = callback_config_error("arize", {var: bad}) + assert error is not None and var in error and repr(bad) in error + + +@pytest.mark.parametrize("var", ["arize_success_sampling_rate", "arize_error_sampling_rate"]) +@pytest.mark.parametrize("good", ["0", "1", "0.25", "", "None"]) +def test_callback_config_error_accepts_in_range_arize_sampling_rate(var, good): + assert callback_config_error("arize", {var: good}) is None + + +def test_arize_sampling_rate_rejected_on_non_arize_callback(): + error = callback_config_error("langfuse", {"arize_success_sampling_rate": "0.5"}) + assert error is not None + assert "applies to the arize callback only" in error + assert callback_config_error("arize", {"arize_success_sampling_rate": "0.5"}) is None + + +def test_arize_sampling_rates_are_not_family_credentials(): + """The rates choose what the Arize family exports, not where it sends, so an + entry that repeats or adds a rate next to a stored Arize entry is not the + credential-redirect shape cross_entry_family_error rejects.""" + stored = [{"arize_api_key": "k1", "arize_success_sampling_rate": "0.5"}] + assert cross_entry_family_error({"arize_success_sampling_rate": "0.1"}, stored) is None + assert cross_entry_family_error({"arize_error_sampling_rate": "0.5"}, stored) is None diff --git a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py index 2a0ebf9492a..0f64ef2b4ca 100644 --- a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py +++ b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py @@ -47,6 +47,7 @@ _EXPECTED_CONFIG_SYNCED_TABLE_NAMES = frozenset( "litellm_proxymodeltable", "litellm_searchtoolstable", "litellm_ssoconfig", + "litellm_uisettings", } ) @@ -702,6 +703,33 @@ async def test_model_repository_write_publishes_via_live_coordination_cache() -> assert json.loads(message) == {"object_type": "litellm_proxymodeltable"} +async def test_ui_settings_write_publishes_via_live_coordination_cache() -> None: + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import _set_redis_usage_cache + from litellm.repositories.table_repositories import UISettingsRepository + + client = _RecordingRedisClient() + prisma_client = MagicMock() + prisma_client.db.litellm_uisettings.upsert = AsyncMock(return_value={"id": "ui_settings"}) + table = UISettingsRepository(prisma_client).table + assert isinstance(table, _PublishOnWriteActions) + + previous_cache = proxy_server.redis_usage_cache + _set_redis_usage_cache(_FakeRedisCache(client)) + try: + await table.upsert( + where={"id": "ui_settings"}, + data={"create": {"id": "ui_settings"}, "update": {"ui_settings": "{}"}}, + ) + finally: + _set_redis_usage_cache(previous_cache) + + assert len(client.published) == 1 + channel, message = client.published[0] + assert channel == CONFIG_SYNC_CHANNEL + assert json.loads(message) == {"object_type": "litellm_uisettings"} + + async def _publish_calls_for_invalidated_param(param_name: str) -> List[Tuple[str, str]]: from litellm.proxy import proxy_server from litellm.proxy.proxy_server import _set_redis_usage_cache diff --git a/tests/test_litellm/proxy/common_utils/test_debug_utils.py b/tests/test_litellm/proxy/common_utils/test_debug_utils.py index 163ea530be9..a64c94c2991 100644 --- a/tests/test_litellm/proxy/common_utils/test_debug_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_debug_utils.py @@ -1,16 +1,25 @@ +import json import os import socket +from collections.abc import Iterator, Mapping +from dataclasses import asdict from pathlib import Path import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy import proxy_server +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.bug_report_config import build_proxy_bug_report from litellm.proxy.common_utils.debug_utils import ( PSUTIL_MISSING_ERROR, _ProcFilesystemProcess, _summary_process_memory, get_memory_summary, ) +from litellm.proxy.common_utils.debug_utils import router as debug_router PAGE_SIZE = 4096 STATM_SIZE_PAGES = 100_000 @@ -67,3 +76,73 @@ async def test_memory_summary_names_the_host_and_worker_that_answered() -> None: assert summary["hostname"] == socket.gethostname() assert summary["worker_pid"] == os.getpid() assert summary["memory"]["ram_usage_mb"] > 0 + + +HOSTILE_CONFIG: Mapping[str, object] = { + "model_list": [ + { + "model_name": "acme-prod-gpt4", + "litellm_params": { + "model": "azure/acme-gpt4o-deployment", + "api_base": "https://acme-eastus.openai.azure.com", + "api_key": "sk-live-secret-1", + }, + } + ], + "litellm_settings": {"drop_params": True, "callbacks": ["langfuse", "acme_hooks.audit_logger"]}, +} + +HOSTILE_GENERAL_SETTINGS: Mapping[str, object] = { + "master_key": "sk-live-secret-master", + "database_url": "postgres://user:hunter2@10.0.0.7/litellm", + "store_model_in_db": True, +} + +HOSTILE_STRINGS = ("acme", "sk-live-secret", "hunter2", "10.0.0.7", "azure.com") + + +@pytest.fixture +def hostile_proxy_config(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + previous_config = proxy_server.proxy_config.get_config_state() + proxy_server.proxy_config.update_config_state(config=HOSTILE_CONFIG) + monkeypatch.setattr(proxy_server, "general_settings", dict(HOSTILE_GENERAL_SETTINGS)) + yield + proxy_server.proxy_config.update_config_state(config=previous_config) + + +def _debug_client(caller: UserAPIKeyAuth) -> TestClient: + app = FastAPI() + app.include_router(debug_router) + app.dependency_overrides[user_api_key_auth] = lambda: caller + return TestClient(app) + + +@pytest.mark.parametrize( + "caller", + [ + UserAPIKeyAuth(), + UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER), + UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY), + ], +) +@pytest.mark.usefixtures("hostile_proxy_config") +def test_debug_report_refuses_everyone_but_proxy_admins(caller: UserAPIKeyAuth) -> None: + response = _debug_client(caller).get("/debug/report") + + assert response.status_code == 403, response.text + assert "litellm_version" not in response.text + + +@pytest.mark.usefixtures("hostile_proxy_config") +def test_debug_report_returns_what_the_bug_report_link_carries_and_nothing_from_the_operator() -> None: + response = _debug_client(UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)).get("/debug/report") + + assert response.status_code == 200, response.text + assert response.json() == json.loads(json.dumps(asdict(build_proxy_bug_report(RuntimeError("boom")).environment))) + assert response.json()["config_lines"] == [ + "general_settings.store_model_in_db = true", + "litellm_settings.drop_params = true", + "litellm_settings.callbacks = [langfuse]", + "model_list[*].provider = [azure]", + ] + assert not any(hostile in response.text for hostile in HOSTILE_STRINGS), response.text diff --git a/tests/test_litellm/proxy/common_utils/test_error_body_call_id.py b/tests/test_litellm/proxy/common_utils/test_error_body_call_id.py new file mode 100644 index 00000000000..8872b5de397 --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_error_body_call_id.py @@ -0,0 +1,35 @@ +import pytest + +from litellm.proxy._types import ConfigGeneralSettings +from litellm.proxy.common_utils.error_body_call_id import error_body_call_id, with_call_id + + +@pytest.mark.parametrize( + "general_settings, call_id, expected", + [ + ({"include_call_id_in_error_body": True}, "call-1", "call-1"), + ({"include_call_id_in_error_body": True}, None, None), + ({"include_call_id_in_error_body": True}, "", None), + ({"include_call_id_in_error_body": False}, "call-1", None), + ({"include_call_id_in_error_body": "true"}, "call-1", None), + ({}, "call-1", None), + ], +) +def test_only_the_boolean_opt_in_with_a_real_id_yields_a_body_call_id(general_settings, call_id, expected): + """The setting is off by default and only a literal true turns it on; without an id + there is nothing to copy, so the body must never get a fabricated one.""" + assert error_body_call_id(general_settings, call_id) == expected + + +def test_with_call_id_appends_the_key_without_touching_the_input(): + error = {"message": "bad input", "type": "invalid_request_error", "param": None, "code": "400"} + + assert with_call_id(error, "call-1") == {**error, "litellm_call_id": "call-1"} + assert with_call_id(error, None) == error + assert "litellm_call_id" not in error + + +def test_the_setting_name_is_a_config_general_settings_field(): + """The yaml key the docs name and the key the runtime reads must be the same field.""" + assert ConfigGeneralSettings.model_validate({"include_call_id_in_error_body": True}).include_call_id_in_error_body + assert ConfigGeneralSettings.model_validate({}).include_call_id_in_error_body is None diff --git a/tests/test_litellm/proxy/common_utils/test_registry_read_through.py b/tests/test_litellm/proxy/common_utils/test_registry_read_through.py index 6f7c20166c5..ca2ff8bcce1 100644 --- a/tests/test_litellm/proxy/common_utils/test_registry_read_through.py +++ b/tests/test_litellm/proxy/common_utils/test_registry_read_through.py @@ -207,7 +207,9 @@ async def test_get_agent_with_read_through_recovers_agent_by_name(clean_agent_re @pytest.mark.asyncio -async def test_get_agent_with_read_through_returns_none_for_unknown_agent(clean_agent_registry, monkeypatch): +async def test_get_agent_with_read_through_returns_none_for_unknown_agent( + clean_agent_registry, fresh_agent_read_through, monkeypatch +): from unittest.mock import AsyncMock, MagicMock import litellm.proxy.proxy_server as proxy_server diff --git a/tests/test_litellm/proxy/common_utils/test_swagger_utils.py b/tests/test_litellm/proxy/common_utils/test_swagger_utils.py new file mode 100644 index 00000000000..659d2ad2941 --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_swagger_utils.py @@ -0,0 +1,18 @@ +import inspect + +from litellm.exceptions import RateLimitError +from litellm.proxy.common_utils.swagger_utils import ERROR_RESPONSES, _error_description + + +class _ChildWithoutDoc(RateLimitError): + pass + + +def test_error_response_descriptions_carry_no_docstring_indentation(): + assert ERROR_RESPONSES[429]["description"] == inspect.cleandoc(RateLimitError.__doc__ or "") + for response in ERROR_RESPONSES.values(): + assert response["description"] == inspect.cleandoc(response["description"]) + + +def test_error_description_falls_back_to_the_class_name_without_an_own_docstring(): + assert _error_description(_ChildWithoutDoc) == "_ChildWithoutDoc" diff --git a/tests/test_litellm/proxy/conftest.py b/tests/test_litellm/proxy/conftest.py index 8ef5017a952..49dc8d02bdb 100644 --- a/tests/test_litellm/proxy/conftest.py +++ b/tests/test_litellm/proxy/conftest.py @@ -273,3 +273,12 @@ def create_proxy_test_client( # Initialize proxy asyncio.run(initialize(config=config_fp, debug=init_options.get("debug", False))) return TestClient(app) + + +@pytest.fixture +def fresh_agent_read_through(monkeypatch): + from litellm.proxy.common_utils import registry_read_through + + read_through = registry_read_through.RegistryReadThrough(resync=registry_read_through._resync_agents) + monkeypatch.setattr(registry_read_through, "agent_registry_read_through", read_through) + return read_through diff --git a/tests/test_litellm/proxy/db/conftest.py b/tests/test_litellm/proxy/db/conftest.py index bcb7794a20d..4d775142dd5 100644 --- a/tests/test_litellm/proxy/db/conftest.py +++ b/tests/test_litellm/proxy/db/conftest.py @@ -2,14 +2,15 @@ import json import os import signal import sys -import time from collections.abc import Generator from dataclasses import dataclass from pathlib import Path -from typing import Final, Optional +from typing import Optional import pytest +from tests._process_helpers import process_is_gone + DB_ENV_KEYS = ( "IAM_TOKEN_DB_AUTH", "AZURE_POSTGRESQL_AUTH", @@ -35,14 +36,6 @@ DB_ENV_KEYS = ( _db_env_snapshot_key = pytest.StashKey[dict[str, Optional[str]]]() -def _is_zombie(pid: int) -> bool: - try: - stat: Final = Path(f"/proc/{pid}/stat").read_text() - except OSError: - return False - return stat.rpartition(")")[2].split()[0] == "Z" - - def _db_env_snapshot() -> dict[str, Optional[str]]: return {key: os.environ.get(key) for key in DB_ENV_KEYS} @@ -130,24 +123,7 @@ class FakePrismaCli: return [json.loads(line) for line in self.calls_file.read_text().splitlines()] def grandchild_is_gone(self, within_seconds: float) -> bool: - pid: Final = int(self.grandchild_pidfile.read_text()) - deadline: Final = time.monotonic() + within_seconds - while time.monotonic() < deadline: - if os.name != "nt": - try: - reaped_pid, _ = os.waitpid(pid, os.WNOHANG) - if reaped_pid == pid: - return True - except ChildProcessError: - pass - try: - os.kill(pid, 0) - except ProcessLookupError: - return True - if _is_zombie(pid): - return True - time.sleep(0.05) - return False + return process_is_gone(int(self.grandchild_pidfile.read_text()), within_seconds=within_seconds) @pytest.fixture diff --git a/tests/test_litellm/proxy/db/test_pgbouncer.py b/tests/test_litellm/proxy/db/test_pgbouncer.py index bf7df3077ea..c69c8d015a3 100644 --- a/tests/test_litellm/proxy/db/test_pgbouncer.py +++ b/tests/test_litellm/proxy/db/test_pgbouncer.py @@ -122,6 +122,13 @@ class TestPlanPgBouncer: "pgbouncer": "true", } + def test_an_upstream_that_already_disables_prepared_statements_gets_a_single_pgbouncer_flag(self): + pooled: Final = _plan("postgresql://app:pw@db/litellm?connection_limit=5&pgbouncer=true").pooled_url + assert urllib.parse.parse_qsl(urllib.parse.urlsplit(pooled).query) == [ + ("connection_limit", "5"), + ("pgbouncer", "true"), + ] + @pytest.mark.parametrize("hop_param", ["channel_binding=require", "gssencmode=require"]) def test_transport_params_for_the_postgres_hop_stay_off_the_plain_tcp_loopback_url(self, hop_param: str): pooled: Final = _plan(f"postgresql://app:pw@db/litellm?connection_limit=5&{hop_param}").pooled_url diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_mcp.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_mcp.py index 826edab694d..2e3bf760e68 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_mcp.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_mcp.py @@ -827,3 +827,38 @@ class TestCiscoAIDefenseJsonRpcSuccessEnvelope: assert unwrapped is verdict else: assert unwrapped == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ["block", "redact"]) +async def test_cisco_native_hook_through_logging_preserves_sanitized_result(action): + from mcp.types import CallToolResult, TextContent + from litellm.litellm_core_utils.litellm_logging import Logging + + guardrail = _make_guardrail( + inspection_type="mcp", event_hook=["pre_mcp_call", "during_mcp_call"] + ) + verdict = ( + _violation_response(url=MCP_URL) if action == "block" + else _redact_response(sanitized_text="[REDACTED]", url=MCP_URL) + ) + result = CallToolResult( + content=[TextContent(type="text", text="SECRET-1234")], + structured_content={"result": "SECRET-1234"}, + ) + logging_obj = Logging( + model="MCP: probe/search", messages=[], stream=False, call_type="call_mcp_tool", + start_time=datetime.now(), litellm_call_id="cisco-hook", function_id="cisco-hook", + dynamic_success_callbacks=[guardrail], + ) + logging_obj.model_call_details.update({"name": "search", "arguments": {}, "original_response": result}) + with _patch_inspection_post(guardrail, AsyncMock(return_value=verdict)): + returned = await logging_obj.async_post_mcp_tool_call_hook( + kwargs=logging_obj.model_call_details, response_obj=result, + start_time=datetime.now(), end_time=datetime.now(), + ) + assert returned is result + assert "SECRET-1234" not in returned.model_dump_json() + assert returned.is_error is (action == "block") + assert ("Blocked by Cisco AI Defense" if action == "block" else "[REDACTED]") in returned.content[0].text + assert returned.structured_content == {"result": returned.content[0].text} diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index 33614d2eeca..89e72debbf5 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -4,6 +4,7 @@ Tests PII detection and masking for different message formats """ import asyncio +import json from contextlib import asynccontextmanager from unittest.mock import MagicMock, patch @@ -18,7 +19,7 @@ from litellm.proxy.guardrails.guardrail_hooks.presidio import ( ) from litellm.exceptions import GuardrailRaisedException from litellm.types.guardrails import LitellmParams, PiiAction, PiiEntityType -from litellm.types.utils import Choices, Message, ModelResponse +from litellm.types.utils import Choices, Delta, Message, ModelResponse, StreamingChoices from litellm.exceptions import BlockedPiiEntityError @@ -2331,47 +2332,320 @@ async def test_apply_guardrail_masks_on_request(): assert "John Smith" not in result["texts"][0] +def _anthropic_sse(event_type: str, payload: dict) -> bytes: + return f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode() + + +def _anthropic_text_deltas(chunks: list[bytes]) -> list[tuple[int, str]]: + deltas = [] + for line in b"".join(chunks).decode().split("\n"): + if not line.startswith("data: "): + continue + event = json.loads(line[6:]) + if event.get("type") == "content_block_delta" and event["delta"].get("type") == "text_delta": + deltas.append((event["index"], event["delta"]["text"])) + return deltas + + +def _chat_delta_chunk(text: str, finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-out-mask", + choices=[StreamingChoices(index=0, delta=Delta(content=text, role="assistant"), finish_reason=finish_reason)], + created=1, + model="gpt-4", + object="chat.completion.chunk", + ) + + @pytest.mark.asyncio -async def test_apply_to_output_streaming_bytes_only_logs_warning(): +async def test_apply_to_output_streaming_chat_chunks_are_masked_as_one_response(): """ - Regression test: when apply_to_output=True and the stream contains only - bytes chunks (Anthropic native SSE), output masking is skipped. - A warning must be logged so operators are aware. + Structured chat completion chunks are buffered, assembled and masked as a + whole, so a card number split across deltas cannot reach the caller. """ guardrail = _OPTIONAL_PresidioPIIMasking( mock_testing=True, apply_to_output=True, + mock_redacted_text={"text": "my card is "}, + ) + + async def mock_stream(): + yield _chat_delta_chunk("my card is 4111") + yield _chat_delta_chunk(" 1111 1111 1111") + yield _chat_delta_chunk("", finish_reason="stop") + + collected = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={"messages": [{"role": "user", "content": "what is my card"}]}, + ): + collected.append(chunk) + + assert all(isinstance(chunk, ModelResponseStream) for chunk in collected) + joined = "".join(chunk.choices[0].delta.content or "" for chunk in collected) + assert joined == "my card is " + assert collected[-1].choices[0].finish_reason == "stop" + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_bytes_after_chat_chunks_are_passed_through_in_order(): + """ + Once structured chunks have been buffered, a trailing bytes frame belongs to + the same stream and must be forwarded rather than treated as a new SSE stream. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": "hello"}, + ) + trailer = b"data: [DONE]\n\n" + + async def mock_stream(): + yield _chat_delta_chunk("hello", finish_reason="stop") + yield trailer + + collected = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + assert collected[0] == trailer + assert len(collected) == 2 + assert isinstance(collected[1], ModelResponseStream) + assert collected[1].choices[0].delta.content == "hello" + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_masks_text_split_across_deltas(): + """ + Anthropic native /v1/messages streams reach the post_call hook as raw SSE + bytes. Output masking must run over the whole content block so a card + number split across text_delta events cannot reach the caller. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": ""}, ) byte_chunks = [ - b'data: {"type":"content_block_delta","delta":{"text":"Hello"}}\n\n', - b'data: {"type":"content_block_delta","delta":{"text":" world"}}\n\n', + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "4111"}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": " 1111 1111 1111"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {}}), + _anthropic_sse("message_stop", {"type": "message_stop"}), ] async def mock_stream(): for b in byte_chunks: yield b - mock_user_api_key = UserAPIKeyAuth(api_key="test-key") + collected = [] + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + assert all(isinstance(chunk, bytes) for chunk in collected) + joined = b"".join(collected).decode() + assert "4111" not in joined + assert "".join(text for _, text in _anthropic_text_deltas(collected)) == "" + assert joined.count("event: message_start") == 1 + assert joined.count("event: message_stop") == 1 + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_without_pii_are_forwarded_unchanged(): + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": "Hello world"}, + ) + + byte_chunks = [ + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Hello"}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": " world"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {}}), + _anthropic_sse("message_stop", {"type": "message_stop"}), + ] + + async def mock_stream(): + for b in byte_chunks: + yield b collected = [] - with patch("litellm.proxy.guardrails.guardrail_hooks.presidio.verbose_proxy_logger") as mock_logger: + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + assert collected == byte_chunks + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_fail_closed_when_presidio_is_unreachable(): + """ + The raw SSE stream is fully drained before masking, so a Presidio outage + must surface as an error to the caller: replaying the unscanned frames + would hand over whatever PII the model generated. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + presidio_analyzer_api_base="http://127.0.0.1:9", + presidio_anonymizer_api_base="http://127.0.0.1:9", + ) + + byte_chunks = [ + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Hello world"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse("message_stop", {"type": "message_stop"}), + ] + + async def mock_stream(): + for b in byte_chunks: + yield b + + collected = [] + + async def collect_masked_stream(): async for chunk in guardrail.async_post_call_streaming_iterator_hook( - user_api_key_dict=mock_user_api_key, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), response=mock_stream(), request_data={}, ): collected.append(chunk) - # All bytes should be yielded through - assert len(collected) == len(byte_chunks) - for original, received in zip(byte_chunks, collected): - assert original == received + with pytest.raises(Exception, match="Presidio PII analysis failed"): + await collect_masked_stream() - # Warning must be logged about skipped masking - mock_logger.warning.assert_called_once() - warning_msg = mock_logger.warning.call_args[0][0] - assert "Output PII masking was skipped" in warning_msg + assert collected == [] + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_anthropic_sse_bytes_block_action_raises_instead_of_replaying(): + """ + A BLOCK on generated PII must refuse the streaming /v1/messages response the + same way it refuses the non streaming one, not replay the raw frames. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + apply_to_output=True, + mock_testing=False, + presidio_analyzer_api_base="http://test-analyzer/", + presidio_anonymizer_api_base="http://test-anonymizer/", + pii_entities_config={PiiEntityType.CREDIT_CARD: PiiAction.BLOCK}, + ) + + byte_chunks = [ + _anthropic_sse( + "message_start", + {"type": "message_start", "message": {"id": "msg_1", "model": "claude", "content": [], "usage": {}}}, + ), + _anthropic_sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _anthropic_sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "4111 1111 1111 1111"}}, + ), + _anthropic_sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _anthropic_sse("message_stop", {"type": "message_stop"}), + ] + + async def mock_stream(): + for b in byte_chunks: + yield b + + analyzer_hit = [{"entity_type": "CREDIT_CARD", "score": 0.99, "start": 0, "end": 19}] + collected = [] + + async def collect_masked_stream(): + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=mock_stream(), + request_data={}, + ): + collected.append(chunk) + + with patch.object(guardrail, "_get_session_iterator", _make_mock_session_iterator(analyzer_hit)): + with pytest.raises(BlockedPiiEntityError): + await collect_masked_stream() + + assert collected == [] + + +@pytest.mark.asyncio +async def test_apply_to_output_streaming_propagates_upstream_error_when_nothing_was_buffered(): + """ + An upstream guardrail that rejects the stream before the first chunk must + surface as an error to the caller, not as an empty 200 stream. + """ + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + mock_redacted_text={"text": ""}, + ) + + async def failing_stream(): + raise RuntimeError("upstream guardrail rejected the stream") + yield b"" + + with pytest.raises(RuntimeError, match="upstream guardrail rejected the stream"): + async for _ in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=failing_stream(), + request_data={}, + ): + pass @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py b/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py index 76027d6b7e2..7a9d155cd67 100644 --- a/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py +++ b/tests/test_litellm/proxy/hooks/test_key_management_event_hooks.py @@ -701,3 +701,49 @@ class TestKeyUpdatedAuditLogObjectId: assert updated_values["project_id"] is None assert json.loads(audit_row.before_value)["project_id"] == "project-orbit" assert updated_values["max_budget"] == 2000.0 + + +@pytest.mark.asyncio +async def test_key_deleted_hook_writes_audit_log_for_alias_deletion(): + from litellm.proxy._types import ( + KeyRequest, + LiteLLM_AuditLogs, + LiteLLM_VerificationToken, + LitellmTableNames, + UserAPIKeyAuth, + ) + + captured: Final[list[LiteLLM_AuditLogs]] = [] + + async def capture_audit_log(request_data: LiteLLM_AuditLogs) -> None: + captured.append(request_data) + + with ( + patch("litellm.store_audit_logs", True), + patch( + "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + new=capture_audit_log, + ), + patch.object( + KeyManagementEventHooks, + "_delete_virtual_keys_from_secret_manager", + new_callable=AsyncMock, + ), + ): + await KeyManagementEventHooks.async_key_deleted_hook( + data=KeyRequest(key_aliases=["a"]), + keys_being_deleted=[LiteLLM_VerificationToken(token="hashed", key_alias="a")], + response={}, + user_api_key_dict=UserAPIKeyAuth(user_id="admin", token="callertok"), + ) + for _ in range(100): + if captured: + break + await asyncio.sleep(0.01) + + assert len(captured) == 1 + audit_row = captured[0] + assert audit_row.action == "deleted" + assert audit_row.object_id == "hashed" + assert audit_row.table_name == LitellmTableNames.KEY_TABLE_NAME + assert audit_row.changed_by == "admin" diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 4907b4ea054..8c0dcd3383c 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -26,6 +26,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PARALLEL_REQUEST_SLOT_TTL_SECONDS, ParallelSlotAcquisition, RateLimitDescriptor, + RateLimitedModel, RateLimitResponse, RequestRateLimiterStash, _request_stash, @@ -3367,7 +3368,7 @@ async def test_pre_call_hook_keeps_internal_stash_out_of_request_body(): stash = get_request_stash() assert stash is not None assert stash.reserved_tokens > 0 - assert stash.reserved_model == "gpt-4o-mini" + assert stash.reserved_model == RateLimitedModel(requested="gpt-4o-mini", group="gpt-4o-mini") assert stash.reserved_scopes == frozenset({("api_key", _api_key)}) @@ -6183,10 +6184,10 @@ async def test_configured_estimate_blocks_the_overrun_the_static_floor_admits(mo assert await admitted({"default_estimated_output_tokens": 3000}) == 2 -def test_internal_call_origin_success_ops_are_skipped(): - """Internal sub-calls (auto-router classifier, shadow eval shadow/judge) bill spend - to the caller's key but must not consume its TPM counters: the same kwargs charge - ops without the origin stamp and none with it.""" +@pytest.mark.parametrize("origin", ["shadow_eval_judge", "autorouter_compaction"]) +@pytest.mark.parametrize("rate_limit_type", ["input", "output", "total"]) +def test_internal_call_origin_success_ops_are_skipped(origin, rate_limit_type): + """Foreground compaction charges the same scopes as ordinary caller traffic.""" handler = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(DualCache()) ) @@ -6202,23 +6203,27 @@ def test_internal_call_origin_success_ops_are_skipped(): def _kwargs(metadata: Dict[str, Any]) -> Dict[str, Any]: return { "standard_logging_object": { - "metadata": {"user_api_key_hash": hash_token("sk-internal-origin")} + "metadata": { + "user_api_key_hash": hash_token("sk-internal-origin"), + "user_api_key_team_id": "compaction-team", + "user_api_key_project_id": "compaction-project", + } }, "litellm_params": {"metadata": metadata}, "model": "gpt-4o-mini", } charged = handler._build_success_event_pipeline_operations( - kwargs=_kwargs({}), response_obj=response, rate_limit_type="output" + kwargs=_kwargs({}), response_obj=response, rate_limit_type=rate_limit_type ) skipped = handler._build_success_event_pipeline_operations( - kwargs=_kwargs({INTERNAL_CALL_ORIGIN_METADATA_KEY: "shadow_eval_judge"}), + kwargs=_kwargs({INTERNAL_CALL_ORIGIN_METADATA_KEY: origin}), response_obj=response, - rate_limit_type="output", + rate_limit_type=rate_limit_type, ) assert charged - assert skipped == [] + assert skipped == (charged if origin == "autorouter_compaction" else []) def _conflicting_budget_bodies() -> Dict[str, Dict[str, object]]: @@ -6961,3 +6966,187 @@ def test_rate_limit_error_reports_reset_time_in_utc_on_a_non_utc_proxy(process_t "Rate limit exceeded for api_key: sk-test. Limit type: requests. " f"Current limit: 2, Remaining: 0. Limit resets at: {expected_reset}" ) + + +def _resolve_alias_to_target(model: str) -> str | None: + return "target" if model == "alias" else None + + +async def _rpm_request(handler: _PROXY_MaxParallelRequestsHandler, cache: DualCache, auth: UserAPIKeyAuth, model: str) -> None: + await handler.async_pre_call_hook(user_api_key_dict=auth, cache=cache, data={"model": model}, call_type="acompletion") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("first_name, second_name", [("target", "alias"), ("alias", "target")]) +async def test_model_group_alias_shares_deployment_default_rpm_bucket_with_its_target( + monkeypatch: pytest.MonkeyPatch, first_name: str, second_name: str +) -> None: + import litellm.proxy.proxy_server as proxy_server + + router: Final = Router( + model_list=[ + { + "model_name": "target", + "litellm_params": {"model": "openai/gpt-test", "api_key": "test-key", "default_api_key_rpm_limit": 2}, + "model_info": {"id": "target-deployment"}, + } + ], + model_group_alias={"alias": "target"}, + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache)) + auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-alias-default")) + + await _rpm_request(handler, cache, auth, first_name) + await _rpm_request(handler, cache, auth, first_name) + with pytest.raises(HTTPException) as exc: + await _rpm_request(handler, cache, auth, second_name) + + assert exc.value.status_code == 429 + assert "model_per_key" in str(exc.value.detail) + assert f"{auth.api_key}:target" in str(exc.value.detail) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("first_name, second_name", [("target", "alias"), ("alias", "target")]) +@pytest.mark.parametrize( + "limits, counter_scope", + [ + ({"metadata": {"model_rpm_limit": {"target": 1}}}, "model_per_key"), + ( + { + "team_id": "t", + "metadata": {"model_rpm_limit": {"other-model": 100}}, + "team_metadata": {"model_rpm_limit": {"target": 1}}, + }, + "model_per_team", + ), + ({"org_id": "o", "organization_metadata": {"model_rpm_limit": {"target": 1}}}, "model_per_organization"), + ({"project_id": "p", "project_metadata": {"model_rpm_limit": {"target": 1}}}, "model_per_project"), + ], + ids=["key_metadata", "team_metadata", "organization_metadata", "project_metadata"], +) +async def test_model_group_alias_shares_metadata_model_rpm_bucket_with_its_target( + limits: dict[str, object], counter_scope: str, first_name: str, second_name: str +) -> None: + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(cache), model_group_resolver=_resolve_alias_to_target + ) + auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-alias-metadata"), **limits) + + await _rpm_request(handler, cache, auth, first_name) + with pytest.raises(HTTPException) as exc: + await _rpm_request(handler, cache, auth, second_name) + + assert exc.value.status_code == 429 + assert counter_scope in str(exc.value.detail) + assert ":target" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_model_rpm_limit_keyed_by_the_alias_name_still_limits_alias_requests_only() -> None: + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(cache), model_group_resolver=_resolve_alias_to_target + ) + auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-alias-keyed"), metadata={"model_rpm_limit": {"alias": 1}}) + + await _rpm_request(handler, cache, auth, "alias") + with pytest.raises(HTTPException) as exc: + await _rpm_request(handler, cache, auth, "alias") + assert exc.value.status_code == 429 + assert "model_per_key" in str(exc.value.detail) + + await _rpm_request(handler, cache, auth, "target") + + +@pytest.mark.parametrize( + "key_metadata, charges_team_model_pool", + [({}, True), ({"model_tpm_limit": {"target": 500}}, False)], + ids=["no_key_override", "key_owns_target_tpm_limit"], +) +def test_success_tpm_accounting_charges_the_alias_target_bucket( + key_metadata: dict[str, object], charges_team_model_pool: bool +) -> None: + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache()), model_group_resolver=_resolve_alias_to_target + ) + response: Final = ModelResponse( + id="alias-tpm", + object="chat.completion", + created=int(datetime.now().timestamp()), + model="alias", + usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), + choices=[], + ) + kwargs: Final = { + "standard_logging_object": { + "metadata": {"user_api_key_hash": hash_token("sk-alias-tpm"), "user_api_key_team_id": "t"} + }, + "litellm_params": { + "metadata": { + "model_group": "alias", + "user_api_key_metadata": key_metadata, + "user_api_key_team_metadata": {"model_tpm_limit": {"target": 500}}, + } + }, + "model": "alias", + } + + ops: Final = handler._build_success_event_pipeline_operations( + kwargs=kwargs, response_obj=response, rate_limit_type="output" + ) + + charged_keys: Final = {op["key"] for op in ops} + assert handler.create_rate_limit_keys("model_per_key", f"{hash_token('sk-alias-tpm')}:target", "tokens") in charged_keys + assert not any(":alias" in key for key in charged_keys) + team_pool_key: Final = handler.create_rate_limit_keys("model_per_team", "t:target", "tokens") + assert (team_pool_key in charged_keys) is charges_team_model_pool + + +@pytest.mark.asyncio +async def test_success_tpm_accounting_keeps_the_admission_target_after_an_alias_reload() -> None: + alias_map: Final[dict[str, str]] = {"alias": "target-a"} + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(cache), model_group_resolver=alias_map.get + ) + key_metadata: Final = {"model_tpm_limit": {"target-a": 1000, "target-b": 1000}} + auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-alias-reload"), metadata=key_metadata) + + await handler.async_pre_call_hook( + user_api_key_dict=auth, + cache=cache, + data={"model": "alias", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10}, + call_type="acompletion", + ) + stash: Final = get_request_stash() + assert stash is not None + assert stash.reserved_model == RateLimitedModel(requested="alias", group="target-a") + assert stash.reserved_tokens > 0 + + alias_map["alias"] = "target-b" + response: Final = ModelResponse( + id="alias-reload", + object="chat.completion", + created=int(datetime.now().timestamp()), + model="alias", + usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), + choices=[], + ) + kwargs: Final = { + "standard_logging_object": {"metadata": {"user_api_key_hash": auth.api_key}}, + "litellm_params": {"metadata": {"model_group": "alias", "user_api_key_metadata": key_metadata}}, + "model": "alias", + } + + ops: Final = handler._build_success_event_pipeline_operations( + kwargs=kwargs, response_obj=response, rate_limit_type="total" + ) + + admission_bucket: Final = handler.create_rate_limit_keys("model_per_key", f"{auth.api_key}:target-a", "tokens") + charged: Final = {op["key"]: op["increment_value"] for op in ops} + assert charged[admission_bucket] == 150 - stash.reserved_tokens + assert not any(":target-b" in key for key in charged) diff --git a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py index bdaca9ffc2d..e6795bb22f3 100644 --- a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py +++ b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py @@ -25,6 +25,7 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY, + RateLimitedModel, _AUDIO_BYTES_PER_TOKEN, _PROXY_MaxParallelRequestsHandler_v3 as RateLimitHandler, ) @@ -308,7 +309,7 @@ async def test_model_scope_refund_targets_reserved_model(rate_limiter): stash = get_or_create_request_stash() stash.reserved_tokens = 100 - stash.reserved_model = reserved_model + stash.reserved_model = RateLimitedModel(requested=reserved_model, group=reserved_model) stash.reserved_scopes = frozenset({("model_per_team", f"{team_id}:{reserved_model}")}) mock_kwargs = { diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index 75545a574e3..2d1049b143e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -1,8 +1,11 @@ +import asyncio import hashlib import json +import logging from datetime import datetime, timezone from types import SimpleNamespace from typing import Final +from unittest.mock import AsyncMock, MagicMock import httpx import pytest @@ -17,12 +20,15 @@ from litellm.proxy._types import ( LiteLLM_UserTableFiltered, LitellmUserRoles, NewUserRequest, + ProxyErrorTypes, ProxyException, UpdateUserRequest, UserAPIKeyAuth, ) from litellm.proxy.management_endpoints.internal_user_endpoints import ( LiteLLM_UserTableWithKeyCount, + _authorize_user_list_request, + _resolve_org_filter_for_user_search, _resolve_user_email_metadata, _update_internal_user_params, get_user_key_counts, @@ -4653,3 +4659,183 @@ async def test_delete_user_evicts_cached_user_rows(mocker: MockerFixture) -> Non assert await cache.async_get_cache(key=deleted.user_id, model_type=LiteLLM_UserTable) is None assert await cache.async_get_cache(key=survivor.user_id, model_type=LiteLLM_UserTable) == survivor broadcast.assert_awaited_once_with(cache_key=deleted.user_id) + + +_DB_OUTAGE_503_BODY: Final = { + "error": { + "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", + "type": "no_db_connection", + "param": "None", + "code": "503", + } +} + + +def _user_read_raising(mocker: MockerFixture, error: Exception) -> tuple[MagicMock, MagicMock]: + prisma_client = MagicMock() + prisma_client.db.litellm_usertable.find_unique = AsyncMock(side_effect=error) + cache = MagicMock() + cache.async_get_cache = AsyncMock(return_value=None) + cache.async_set_cache = AsyncMock() + mocker.patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True) + return prisma_client, cache + + +def _db_unavailable_fallback_identity(route: str) -> UserAPIKeyAuth: + from litellm.proxy.auth.auth_exception_handler import DB_UNAVAILABLE_FALLBACK_USER_ID + + return UserAPIKeyAuth( + key_name="failed-to-connect-to-db", + token="failed-to-connect-to-db", + user_id=DB_UNAVAILABLE_FALLBACK_USER_ID, + user_role=LitellmUserRoles.INTERNAL_USER, + request_route=route, + ) + + +@pytest.mark.asyncio +async def test_authorize_user_list_request_propagates_a_db_outage_instead_of_answering_403(mocker): + prisma_client, cache = _user_read_raising(mocker, httpx.ConnectError("All connection attempts failed")) + + with pytest.raises(httpx.ConnectError): + await _authorize_user_list_request( + user_api_key_dict=_db_unavailable_fallback_identity("/user/list"), + organization_ids=None, + prisma_client=prisma_client, + user_api_key_cache=cache, + proxy_logging_obj=None, + ) + + +@pytest.mark.asyncio +async def test_resolve_org_filter_for_user_search_propagates_a_db_outage_instead_of_answering_403(mocker): + prisma_client, cache = _user_read_raising(mocker, httpx.ConnectError("All connection attempts failed")) + mocker.patch( + "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.get_ui_settings_cached", + return_value={"scope_user_search_to_org": True}, + ) + + with pytest.raises(httpx.ConnectError): + await _resolve_org_filter_for_user_search( + user_api_key_dict=_db_unavailable_fallback_identity("/user/filter/ui"), + team_id=None, + prisma_client=prisma_client, + user_api_key_cache=cache, + proxy_logging_obj=None, + ) + + +@pytest.mark.asyncio +async def test_ui_view_users_answers_a_db_outage_as_503_no_db_connection_not_as_its_own_500(mocker, caplog): + prisma_client, cache = _user_read_raising(mocker, httpx.ConnectError("All connection attempts failed")) + mocker.patch( + "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.get_ui_settings_cached", + return_value={"scope_user_search_to_org": True}, + ) + proxy_logging_obj = MagicMock() + proxy_logging_obj.service_logging_obj.async_service_failure_hook = AsyncMock() + mocker.patch("litellm.proxy.proxy_server.prisma_client", prisma_client) + mocker.patch("litellm.proxy.proxy_server.user_api_key_cache", cache) + mocker.patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_obj) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"), pytest.raises(ProxyException) as raised: + await ui_view_users( + user_api_key_dict=_db_unavailable_fallback_identity("/user/filter/ui"), + user_id=None, + user_email="lit", + team_id=None, + page=1, + page_size=50, + ) + + assert raised.value.code == "503" + assert raised.value.type == ProxyErrorTypes.no_db_connection + assert isinstance(raised.value.__cause__, httpx.ConnectError) + outage_logs: Final = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and "ConnectError" in r.getMessage()] + assert outage_logs == ["Database unavailable during user search: ConnectError"] + + +@pytest.mark.parametrize( + ("route", "params"), + [("/user/list", {}), ("/user/filter/ui", {"user_email": "lit"})], + ids=["user_list", "user_filter_ui"], +) +def test_user_routes_answer_503_no_db_connection_when_the_callers_user_read_hits_a_db_outage( + mocker, route: str, params: dict[str, str] +): + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + prisma_client, cache = _user_read_raising(mocker, httpx.ConnectError("All connection attempts failed")) + mocker.patch( + "litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints.get_ui_settings_cached", + return_value={"scope_user_search_to_org": True}, + ) + mocker.patch("litellm.proxy.proxy_server.prisma_client", prisma_client) + mocker.patch("litellm.proxy.proxy_server.user_api_key_cache", cache) + app.dependency_overrides[user_api_key_auth] = lambda: _db_unavailable_fallback_identity(route) + try: + response = TestClient(app, raise_server_exceptions=False).get(route, params=params) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 503, response.text + assert response.json() == _DB_OUTAGE_503_BODY + + +@pytest.mark.asyncio +async def test_delete_user_writes_deleted_audit_log_for_user_keys(mocker): + from litellm.proxy._types import ( + DeleteUserRequest, + LiteLLM_VerificationToken, + LitellmTableNames, + UserAPIKeyAuth, + ) + from litellm.proxy.management_endpoints.internal_user_endpoints import delete_user + + mock_prisma_client = mocker.MagicMock() + + mock_user_row = mocker.MagicMock() + mock_user_row.user_id = "doomed-user" + mock_user_row.user_email = "doomed@example.com" + mock_user_row.teams = [] + mock_user_row.model_dump_json.return_value = "{}" + mock_user_row.model_dump.return_value = {"user_id": "doomed-user", "user_email": "doomed@example.com", "teams": []} + + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(return_value=mock_user_row) + mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(return_value=[]) + + user_key = LiteLLM_VerificationToken(token="hashed-user-key", user_id="doomed-user") + mock_prisma_client.db.litellm_verificationtoken.find_many = mocker.AsyncMock(return_value=[user_key]) + mock_prisma_client.db.litellm_verificationtoken.delete_many = mocker.AsyncMock(return_value=1) + mock_prisma_client.db.litellm_invitationlink.delete_many = mocker.AsyncMock(return_value=0) + mock_prisma_client.db.litellm_organizationmembership.delete_many = mocker.AsyncMock(return_value=0) + mock_prisma_client.db.litellm_teammembership.delete_many = mocker.AsyncMock(return_value=0) + mock_prisma_client.db.litellm_jwtkeymapping.find_many = mocker.AsyncMock(return_value=[]) + mock_prisma_client.db.litellm_usertable.delete_many = mocker.AsyncMock(return_value=1) + + mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mocker.patch("litellm.store_audit_logs", True) + captured: Final[list] = [] + + async def _capture(request_data): + captured.append(request_data) + + mocker.patch( + "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + new=_capture, + ) + + caller = UserAPIKeyAuth(user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN) + await delete_user(data=DeleteUserRequest(user_ids=["doomed-user"]), user_api_key_dict=caller) + for _ in range(100): + if captured: + break + await asyncio.sleep(0.01) + + key_rows: Final = [r for r in captured if r.table_name == LitellmTableNames.KEY_TABLE_NAME] + assert len(key_rows) == 1 + audit_row: Final = key_rows[0] + assert audit_row.action == "deleted" + assert audit_row.object_id == user_key.token + assert audit_row.changed_by + assert json.loads(audit_row.before_value)["token"] == user_key.token diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 8eaa4901c59..cd929dfeeb3 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -609,10 +609,7 @@ async def test_generate_key_debug_log_never_contains_raw_token(monkeypatch, capl mock_prisma_client.insert_data = AsyncMock(side_effect=_insert_data_side_effect) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) from litellm.proxy._types import GenerateKeyRequest, LitellmUserRoles from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth @@ -1494,18 +1491,12 @@ async def test_list_keys_full_object_returns_lifetime_total_spend(): @pytest.mark.asyncio async def test_get_new_token_with_valid_key(monkeypatch): """Test get_new_token function when provided with a valid key that starts with 'sk-'""" - from unittest.mock import AsyncMock - from litellm.proxy._types import RegenerateKeyRequest from litellm.proxy.management_endpoints.key_management_endpoints import ( get_new_token, ) - # Mock get_ui_settings_cached to return setting disabled (custom keys allowed) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) # Test with valid new_key data = RegenerateKeyRequest(new_key="sk-test1234567890abc") @@ -1517,8 +1508,6 @@ async def test_get_new_token_with_valid_key(monkeypatch): @pytest.mark.asyncio async def test_get_new_token_with_invalid_key(monkeypatch): """Test get_new_token function when provided with an invalid key that doesn't start with 'sk-'""" - from unittest.mock import AsyncMock - from fastapi import HTTPException from litellm.proxy._types import RegenerateKeyRequest @@ -1526,11 +1515,7 @@ async def test_get_new_token_with_invalid_key(monkeypatch): get_new_token, ) - # Mock get_ui_settings_cached to return setting disabled (custom keys allowed) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) # Test with invalid new_key (doesn't start with 'sk-') data = RegenerateKeyRequest(new_key="invalid-key-123") @@ -1546,8 +1531,6 @@ async def test_get_new_token_with_invalid_key(monkeypatch): async def test_get_new_token_rejects_short_new_key(monkeypatch): """Regression test for LIT-4355: a short custom key like sk-99 must be rejected, otherwise the stored key_name (sk-...{last 4 chars}) reveals the entire key.""" - from unittest.mock import AsyncMock - from fastapi import HTTPException from litellm.proxy._types import RegenerateKeyRequest @@ -1555,10 +1538,7 @@ async def test_get_new_token_rejects_short_new_key(monkeypatch): get_new_token, ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) data = RegenerateKeyRequest(new_key="sk-99") @@ -1588,10 +1568,7 @@ async def test_generate_key_fn_rejects_short_custom_key(monkeypatch, short_key): ) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) assert len(short_key) < 16 @@ -1628,10 +1605,7 @@ async def test_generate_key_fn_accepts_custom_key_at_minimum_length(monkeypatch) ) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) custom_key = "sk-abcdefghijklm" assert len(custom_key) == 16 @@ -1649,18 +1623,13 @@ async def test_generate_key_fn_accepts_custom_key_at_minimum_length(monkeypatch) @pytest.mark.asyncio async def test_check_custom_key_allowed_when_disabled(monkeypatch): """_check_custom_key_allowed raises 403 when disable_custom_api_keys is true.""" - from unittest.mock import AsyncMock - from fastapi import HTTPException from litellm.proxy.management_endpoints.key_management_endpoints import ( _check_custom_key_allowed, ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={"disable_custom_api_keys": True}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"disable_custom_api_keys": True}) with pytest.raises(HTTPException) as exc_info: await _check_custom_key_allowed("sk-custom-key-123") @@ -1672,16 +1641,11 @@ async def test_check_custom_key_allowed_when_disabled(monkeypatch): @pytest.mark.asyncio async def test_check_custom_key_allowed_when_enabled(monkeypatch): """_check_custom_key_allowed does nothing when disable_custom_api_keys is false.""" - from unittest.mock import AsyncMock - from litellm.proxy.management_endpoints.key_management_endpoints import ( _check_custom_key_allowed, ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={"disable_custom_api_keys": False}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"disable_custom_api_keys": False}) # Should not raise await _check_custom_key_allowed("sk-custom-key-123") @@ -1690,35 +1654,133 @@ async def test_check_custom_key_allowed_when_enabled(monkeypatch): @pytest.mark.asyncio async def test_check_custom_key_allowed_when_unset(monkeypatch): """_check_custom_key_allowed does nothing when setting is not present.""" - from unittest.mock import AsyncMock - from litellm.proxy.management_endpoints.key_management_endpoints import ( _check_custom_key_allowed, ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) # Should not raise await _check_custom_key_allowed("sk-custom-key-123") @pytest.mark.asyncio -async def test_check_custom_key_allowed_none_key_always_passes(monkeypatch): - """_check_custom_key_allowed does nothing when key is None, even if setting is on.""" - from unittest.mock import AsyncMock +async def test_check_custom_key_allowed_honours_the_config_file(monkeypatch): + """A config-file general_settings.disable_custom_api_keys is enforced with no stored UI row.""" + from fastapi import HTTPException + from litellm.proxy.config_resolvers import SettingsStore from litellm.proxy.management_endpoints.key_management_endpoints import ( _check_custom_key_allowed, ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={"disable_custom_api_keys": True}), + general_settings = SettingsStore("general_settings") + general_settings.load_yaml({"disable_custom_api_keys": True}) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", general_settings) + + with pytest.raises(HTTPException) as exc_info: + await _check_custom_key_allowed("sk-custom-key-123456") + + assert exc_info.value.status_code == 403 + + +@pytest.mark.parametrize( + ("config_value", "blocked"), + [ + (True, True), + ("true", True), + ("True", True), + (1, True), + (False, False), + ("false", False), + ("False", False), + (0, False), + ], +) +@pytest.mark.asyncio +async def test_check_custom_key_allowed_coerces_a_non_bool_config_value(monkeypatch, config_value, blocked): + """A YAML value that is not a bare bool, such as a quoted "true", still decides the gate.""" + from fastapi import HTTPException + + from litellm.proxy.config_resolvers import SettingsStore + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _check_custom_key_allowed, ) + general_settings = SettingsStore("general_settings") + general_settings.load_yaml({"disable_custom_api_keys": config_value}) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", general_settings) + + rejected = False + try: + await _check_custom_key_allowed("sk-custom-key-123456") + except HTTPException as e: + rejected = e.status_code == 403 + + assert rejected is blocked + + +@pytest.mark.asyncio +async def test_check_custom_key_allowed_config_file_beats_the_stored_ui_row(monkeypatch): + """The config file owns the flag, so a stored UI row saying false cannot re-open custom keys.""" + from fastapi import HTTPException + + from litellm.proxy.config_resolvers import SettingsStore + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _check_custom_key_allowed, + ) + from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( + apply_runtime_general_settings_flags, + ) + + general_settings = SettingsStore("general_settings") + general_settings.load_yaml({"disable_custom_api_keys": True}) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", general_settings) + + apply_runtime_general_settings_flags({"disable_custom_api_keys": False}) + + with pytest.raises(HTTPException) as exc_info: + await _check_custom_key_allowed("sk-custom-key-123456") + + assert exc_info.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_check_custom_key_allowed_picks_up_a_ui_write_without_the_serving_pod(monkeypatch): + """A pod that never served the PATCH enforces the new value after its own settings sync.""" + from fastapi import HTTPException + + from litellm.proxy.config_resolvers import SettingsStore + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _check_custom_key_allowed, + ) + from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( + apply_runtime_general_settings_flags, + ) + + general_settings = SettingsStore("general_settings") + general_settings.load_yaml({}) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", general_settings) + + await _check_custom_key_allowed("sk-custom-key-123456") + + apply_runtime_general_settings_flags({"disable_custom_api_keys": True}) + + with pytest.raises(HTTPException) as exc_info: + await _check_custom_key_allowed("sk-custom-key-123456") + + assert exc_info.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_check_custom_key_allowed_none_key_always_passes(monkeypatch): + """_check_custom_key_allowed does nothing when key is None, even if setting is on.""" + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _check_custom_key_allowed, + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"disable_custom_api_keys": True}) + # Should not raise — None means auto-generate await _check_custom_key_allowed(None) @@ -1726,8 +1788,6 @@ async def test_check_custom_key_allowed_none_key_always_passes(monkeypatch): @pytest.mark.asyncio async def test_get_new_token_rejected_when_custom_keys_disabled(monkeypatch): """get_new_token raises 403 when new_key is set and disable_custom_api_keys is true.""" - from unittest.mock import AsyncMock - from fastapi import HTTPException from litellm.proxy._types import RegenerateKeyRequest @@ -1735,10 +1795,7 @@ async def test_get_new_token_rejected_when_custom_keys_disabled(monkeypatch): get_new_token, ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={"disable_custom_api_keys": True}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"disable_custom_api_keys": True}) data = RegenerateKeyRequest(new_key="sk-custom-regen-key") @@ -1751,17 +1808,12 @@ async def test_get_new_token_rejected_when_custom_keys_disabled(monkeypatch): @pytest.mark.asyncio async def test_get_new_token_auto_generates_when_custom_keys_disabled(monkeypatch): """get_new_token auto-generates a key when new_key is None, even if setting is on.""" - from unittest.mock import AsyncMock - from litellm.proxy._types import RegenerateKeyRequest from litellm.proxy.management_endpoints.key_management_endpoints import ( get_new_token, ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_ui_settings_cached", - AsyncMock(return_value={"disable_custom_api_keys": True}), - ) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"disable_custom_api_keys": True}) data = RegenerateKeyRequest() # no new_key result = await get_new_token(data) @@ -3037,6 +3089,50 @@ async def test_update_key_by_alias_only(monkeypatch): assert result["key"] == hashed_token +@pytest.mark.asyncio +async def test_update_key_changed_alias_must_match_key_alias_pattern(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + update_key_fn, + ) + + monkeypatch.setattr(litellm, "key_alias_pattern", r"^[a-z0-9]+(-[a-z0-9]+)*$") + hashed_token = "0d62f396c1317066f55a96086517047c737087c61eb2bf016b72e6298927b15b" + key_in_db = LiteLLM_VerificationToken(token=hashed_token, key_alias="Legacy Alias", user_id="test-user") + + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=key_in_db) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[key_in_db]) + mock_prisma_client.db.litellm_verificationtoken.find_first = AsyncMock(return_value=None) + mock_prisma_client.update_data = AsyncMock(return_value={"data": {"max_budget": 50.0}}) + _setup_update_key_mocks(monkeypatch, mock_prisma_client) + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin", user_id="admin-user" + ) + + with pytest.raises(ProxyException) as exc_info: + await update_key_fn( + request=MagicMock(), + data=UpdateKeyRequest(key=hashed_token, key_alias="Prod Key"), + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + assert str(exc_info.value.code) == "400" + assert "key_alias_pattern" in str(exc_info.value.message) + mock_prisma_client.update_data.assert_not_awaited() + + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", + return_value=None, + ): + await update_key_fn( + request=MagicMock(), + data=UpdateKeyRequest(key=hashed_token, key_alias="Legacy Alias", max_budget=50.0), + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + mock_prisma_client.update_data.assert_awaited_once() + + @pytest.mark.asyncio async def test_update_key_by_alias_not_found_returns_404(monkeypatch): """ @@ -10510,6 +10606,10 @@ class TestValidateKeyAliasFormat: def reset_key_alias_flag(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "enable_key_alias_format_validation", False) + @pytest.fixture(autouse=True) + def reset_key_alias_pattern(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "key_alias_pattern", None) + def test_validation_skipped_when_flag_disabled(self): """When enable_key_alias_format_validation is False (default), no charset/length validation occurs.""" from litellm.proxy.management_endpoints.key_management_endpoints import ( @@ -10592,6 +10692,67 @@ class TestValidateKeyAliasFormat: assert str(exc.value.code) == "400" assert "Invalid key_alias format" in str(exc.value.message) + def test_configured_pattern_applies_with_flag_off(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _validate_key_alias_format, + ) + + monkeypatch.setattr(litellm, "key_alias_pattern", r"^[a-z0-9]+(-[a-z0-9]+)*$") + with pytest.raises(ProxyException) as exc: + _validate_key_alias_format("Prod Key") + assert str(exc.value.code) == "400" + assert exc.value.param == "key_alias" + assert "key_alias_pattern" in str(exc.value.message) + assert r"^[a-z0-9]+(-[a-z0-9]+)*$" in str(exc.value.message) + assert _validate_key_alias_format("prod-key-001") is None + assert _validate_key_alias_format(None) is None + + def test_configured_pattern_must_match_the_whole_alias(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _validate_key_alias_format, + ) + + monkeypatch.setattr(litellm, "key_alias_pattern", r"team-[a-z]+") + _validate_key_alias_format("team-search") + for partial_match in ("team-search-2", "xteam-search"): + with pytest.raises(ProxyException): + _validate_key_alias_format(partial_match) + + def test_configured_pattern_replaces_the_builtin_rule(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _validate_key_alias_format, + ) + + monkeypatch.setattr(litellm, "enable_key_alias_format_validation", True) + monkeypatch.setattr(litellm, "key_alias_pattern", r"^[a-z ]+$") + _validate_key_alias_format("alias with spaces") + with pytest.raises(ProxyException) as exc: + _validate_key_alias_format("Uppercase") + assert "key_alias_pattern" in str(exc.value.message) + + def test_configured_pattern_keeps_the_baseline_safety_check(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _validate_key_alias_format, + ) + + monkeypatch.setattr(litellm, "key_alias_pattern", r".*") + with pytest.raises(ProxyException) as exc: + _validate_key_alias_format("../../../other-app/creds") + assert str(exc.value.code) == "400" + assert "key_alias_pattern" not in str(exc.value.message) + + def test_configured_pattern_bounds_the_alias_length(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _validate_key_alias_format, + ) + + monkeypatch.setattr(litellm, "key_alias_pattern", r"^[a-z]+$") + _validate_key_alias_format("a" * 255) + with pytest.raises(ProxyException) as exc: + _validate_key_alias_format("a" * 256) + assert str(exc.value.code) == "400" + assert "at most 255 characters" in str(exc.value.message) + @pytest.mark.asyncio async def test_check_org_key_limits_on_update_within_bounds(): @@ -12682,6 +12843,55 @@ async def test_execute_virtual_key_regeneration_rejects_over_limit_duration(monk assert mock_prisma_client.db.litellm_verificationtoken.update.await_count == 0 +@pytest.mark.asyncio +async def test_execute_virtual_key_regeneration_changed_alias_must_match_key_alias_pattern( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy._types import RegenerateKeyRequest + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _execute_virtual_key_regeneration, + ) + + monkeypatch.setattr(litellm, "key_alias_pattern", r"^[a-z0-9]+(-[a-z0-9]+)*$") + mock_prisma_client = _make_regenerate_mock_prisma() + + with ( + patch( + "litellm.proxy.management_endpoints.key_management_endpoints.get_new_token", + new_callable=AsyncMock, + return_value="sk-newtoken1234ab12", + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._insert_deprecated_key", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._persist_deleted_verification_tokens", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", + new_callable=AsyncMock, + ), + ): + with pytest.raises(ProxyException) as exc_info: + await _execute_virtual_key_regeneration( + prisma_client=mock_prisma_client, + key_in_db=_make_regenerate_existing_key(), + hashed_api_key="abc123", + key="abc123", + data=RegenerateKeyRequest(key_alias="Regenerated Key"), + user_api_key_dict=_make_regenerate_user_api_key_dict(), + litellm_changed_by=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + ) + assert str(exc_info.value.code) == "400" + assert exc_info.value.param == "key_alias" + assert r"^[a-z0-9]+(-[a-z0-9]+)*$" in str(exc_info.value.message) + assert mock_prisma_client.db.litellm_verificationtoken.update.await_count == 0 + + @pytest.mark.asyncio async def test_execute_virtual_key_regeneration_allows_within_limit_duration(monkeypatch): """Regenerate must accept durations within upperbound_key_generate_params.duration.""" diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index c66d490f122..53645e62034 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -7878,3 +7878,65 @@ class TestDeleteMCPGatewaySessions: assert result.terminated_sessions == 2 assert {s.user_id for s in result.sessions} == {"bob"} assert "sk-live-bob" not in result.model_dump_json() + + +class TestGetMcpToolsWireShape: + @pytest.mark.asyncio + async def test_get_mcp_tools_returns_each_tool_in_mcp_wire_spelling(self): + from mcp.types import ListToolsResult, Tool + + add_schema = {"type": "object", "properties": {"a": {"type": "integer"}}, "required": ["a"]} + listed = ListToolsResult( + tools=[Tool(name="add", description="Add", inputSchema=add_schema, outputSchema={"type": "integer"})] + ) + with patch( + "litellm.proxy._experimental.mcp_server.server._list_mcp_tools", + AsyncMock(return_value=listed), + ): + result = await mgmt_endpoints.get_mcp_tools(user_api_key_dict=generate_mock_user_api_key_auth()) + + (tool,) = result["tools"] + assert tool["inputSchema"] == add_schema + assert tool["outputSchema"] == {"type": "integer"} + assert "_meta" in tool + assert not {"input_schema", "output_schema", "meta"} & tool.keys() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("role,expected_status", [ + (LitellmUserRoles.PROXY_ADMIN, 404), + (LitellmUserRoles.INTERNAL_USER, 403), +]) +async def test_config_server_edit_preserves_api_contract_without_creating_rows(role, expected_status): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import MCPServerManager + + manager = MCPServerManager() + server = generate_mock_mcp_server_config_record(server_id="read-only-config") + manager.config_mcp_servers = {server.server_id: server} + original = server.model_dump() + prisma = MagicMock() + prisma.db.litellm_mcpservertable.find_unique = AsyncMock(return_value=None) + prisma.db.litellm_mcpservertable.update = AsyncMock(return_value=None) + with ( + patch.object(mgmt_endpoints, "global_mcp_server_manager", manager), + patch.object(mgmt_endpoints, "get_prisma_client_or_throw", return_value=prisma), + ): + with pytest.raises(HTTPException) as exc: + await mgmt_endpoints.edit_mcp_server( + payload=UpdateMCPServerRequest(server_id=server.server_id, description="UI edit"), + user_api_key_dict=UserAPIKeyAuth(user_id="actor", user_role=role), + ) + + assert exc.value.status_code == expected_status + if role == LitellmUserRoles.PROXY_ADMIN: + assert exc.value.detail == { + "error": f"MCP Server not found, passed server_id={server.server_id}" + } + prisma.db.litellm_mcpservertable.update.assert_awaited_once() + else: + prisma.db.litellm_mcpservertable.update.assert_not_awaited() + prisma.db.litellm_mcpservertable.create.assert_not_called() + prisma.db.litellm_mcpservertable.create_many.assert_not_called() + prisma.tx.assert_not_called() + assert server.model_dump() == original + assert manager.registry == {} diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 5e6c37c41dd..bd252169131 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -4063,6 +4063,294 @@ class TestModelInfoServerDerivedPricingFilter: assert written["access_groups"] == ["prod"] +class TestModelInfoCostMapEchoFilter: + """LIT-5534. ``/model/info`` fills a deployment's ``model_info`` from the cost map (context + limits, mode, provider, supported params, capability flags), and the Admin UI edit form sends + that whole blob back on any save. Only values that still equal the cost-map entry are the + echo; a value the operator changed is a real override and stays.""" + + def test_echoed_cost_map_metadata_is_not_persisted(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("openai/gpt-5.6") + echo = {**entry, "id": "dep-echo-0", "db_model": True, "access_groups": ["prod"]} + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-0"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert info["access_groups"] == ["prod"] + assert set(info).isdisjoint(entry) + assert "max_input_tokens" not in info and "mode" not in info and "supports_vision" not in info, ( + "cost-map metadata must not be persisted from an unchanged /model/info echo" + ) + + def test_an_edited_value_survives_the_echo_filter(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("openai/gpt-5.6") + echo = { + **entry, + "id": "dep-echo-1", + "db_model": True, + "access_groups": ["prod"], + "max_input_tokens": entry["max_input_tokens"] + 1, + "mode": "completion" if entry["mode"] != "completion" else "chat", + } + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-1"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert info["max_input_tokens"] == echo["max_input_tokens"] + assert info["mode"] == echo["mode"] + assert "litellm_provider" not in info + assert "supported_openai_params" not in info + + def test_metadata_without_a_cost_map_key_is_persisted(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + from litellm.types.utils import echoed_cost_map_fields + + entry = litellm.get_model_info("openai/gpt-5.6") + assert echoed_cost_map_fields({"max_input_tokens": entry["max_input_tokens"]}, entry) == () + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-2"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment( + model_info=ModelInfo( + id="dep-echo-2", + max_input_tokens=entry["max_input_tokens"], + mode=entry["mode"], + ) + ), + ) + + info = json.loads(result["model_info"]) + assert info["max_input_tokens"] == entry["max_input_tokens"] + assert info["mode"] == entry["mode"] + + def test_a_stored_mode_survives_an_echoed_save(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("openai/gpt-5.6") + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-3", mode=entry["mode"]), + ) + echo = {**entry, "id": "dep-echo-3", "db_model": True, "access_groups": ["prod"]} + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert info["mode"] == entry["mode"] + assert "max_input_tokens" not in info + + def test_resetting_an_override_to_the_cost_map_value_removes_it(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("openai/gpt-5.6") + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-4", mode="chat", max_input_tokens=2048), + ) + echo = {**entry, "id": "dep-echo-4", "db_model": True, "access_groups": ["staging"]} + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert "max_input_tokens" not in info + assert info["mode"] == "chat" + assert info["access_groups"] == ["staging"] + + def test_reset_is_recognised_after_the_router_registered_the_override(self, monkeypatch: pytest.MonkeyPatch): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + pristine = litellm.get_model_info("openai/gpt-5.6") + polluted = {**pristine, "max_input_tokens": 2048} + monkeypatch.setattr(litellm, "get_model_info", lambda model, **_: polluted) + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-8", max_input_tokens=2048), + ) + echo = {**pristine, "id": "dep-echo-8", "db_model": True} + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert "max_input_tokens" not in info, info + + def test_reset_to_a_remote_catalog_value_that_differs_from_the_bundled_one(self, monkeypatch: pytest.MonkeyPatch): + from types import MappingProxyType + + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + bundled = litellm.get_model_info("openai/gpt-5.6") + remote = {**bundled, "max_input_tokens": bundled["max_input_tokens"] + 1} + remote_catalog = MappingProxyType({remote["key"]: MappingProxyType(remote)}) + monkeypatch.setattr(litellm, "get_model_info", lambda model, **_: {**remote, "max_input_tokens": 2048}) + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-9", max_input_tokens=2048), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**{**remote, "id": "dep-echo-9", "db_model": True})), + loaded_catalog=lambda: remote_catalog, + ) + + info = json.loads(result["model_info"]) + assert "max_input_tokens" not in info, info + + def test_echo_is_compared_against_the_deployments_lookup_not_the_key(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + lookup_pairs: Final = ( + ("openai/gpt-5.6", "gpt-5.6"), + ("openai/gpt-4.1-mini", "gpt-4.1-mini"), + ) + lookup_data: Final = tuple( + (deployment_model, deployment_entry, differing_fields) + for deployment_model, key_model in lookup_pairs + for deployment_entry in (litellm.get_model_info(deployment_model),) + for key_entry in (litellm.get_model_info(key_model),) + for differing_fields in ( + frozenset( + k for k in deployment_entry if k in key_entry and deployment_entry[k] != key_entry[k] + ), + ) + if differing_fields + ) + if not lookup_data: + pytest.skip("No deployment/key cost-map lookup differences are available") + + deployment_model, entry, differing_fields = lookup_data[0] + assert differing_fields + db_model = Deployment( + model_name=deployment_model, + litellm_params=LiteLLM_Params(model=deployment_model), + model_info=ModelInfo(id="dep-echo-5"), + ) + echo = {**entry, "id": "dep-echo-5", "db_model": True, "access_groups": ["prod"]} + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert not frozenset(info).intersection(frozenset(entry) - frozenset(("mode",))) + + def test_base_model_wins_over_litellm_params_model_for_the_lookup(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("azure/gpt-5.6") + db_model = Deployment( + model_name="azure/my-deploy", + litellm_params=LiteLLM_Params(model="azure/my-deploy"), + model_info=ModelInfo(id="dep-echo-6", base_model="azure/gpt-5.6"), + ) + echo = { + **entry, + "id": "dep-echo-6", + "base_model": "azure/gpt-5.6", + "db_model": True, + } + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert not frozenset(info).intersection(frozenset(entry) - frozenset(("mode",))) + assert info["base_model"] == "azure/gpt-5.6" + + def test_encrypted_stored_model_is_decrypted_for_the_lookup(self, monkeypatch): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-1234") + entry = litellm.get_model_info("openai/gpt-5.6") + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model=encrypt_value_helper(value="openai/gpt-5.6")), + model_info=ModelInfo(id="dep-echo-7", mode="chat"), + ) + echo = {**entry, "id": "dep-echo-7", "db_model": True, "access_groups": ["prod"]} + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert not frozenset(info).intersection(frozenset(entry) - frozenset(("mode",))) + assert info["mode"] == "chat" + assert info["access_groups"] == ["prod"] + + class TestUpdateDBModelClearCacheControlInjectionPoints: def test_explicit_null_removes_stored_injection_points(self): from litellm.proxy.management_endpoints.model_management_endpoints import ( diff --git a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py index 3fcda310435..ec1af0518c6 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py @@ -6,20 +6,22 @@ Tests the GET endpoints for router settings and router fields. from collections.abc import Mapping from typing import Any, Final -from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient - from litellm.proxy import proxy_server from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.config_resolvers import SettingsStore from litellm.proxy.management_endpoints.router_settings_endpoints import ( + RouterFieldsResponse, + RouterSettingsResponse, + get_router_fields, get_router_settings, ) -from litellm.proxy.config_resolvers import SettingsStore from litellm.proxy.proxy_server import app from litellm.router import Router +from litellm.types.router import RoutingGroup client = TestClient(app) @@ -157,3 +159,91 @@ class TestRouterSettingsEndpoints: rg_field = next(f for f in response.fields if f.field_name == "routing_groups") assert rg_field.field_value == groups + + @pytest.mark.asyncio + @pytest.mark.parametrize("metadata_only", (True, False)) + async def test_priority_is_advertised_for_groups_only( + self, monkeypatch: pytest.MonkeyPatch, metadata_only: bool + ) -> None: + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr( + proxy_server, + "proxy_config", + _StubProxyConfig(SettingsStore("router_settings"), {}), + ) + admin_user: Final = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-test" + ) + + response: Final[RouterFieldsResponse | RouterSettingsResponse] = ( + await get_router_fields(user_api_key_dict=admin_user) + if metadata_only + else await get_router_settings(user_api_key_dict=admin_user) + ) + + global_options: Final = next( + field.options + for field in response.fields + if field.field_name == "routing_strategy" + ) + assert global_options is not None + assert "priority" not in global_options + assert response.model_dump(mode="json")["routing_group_strategies"] == [*global_options, "priority"] + + @pytest.mark.asyncio + @pytest.mark.parametrize("from_config", (False, True)) + async def test_settings_retains_explicit_model_priorities( + self, monkeypatch: pytest.MonkeyPatch, from_config: bool + ) -> None: + group: Final = RoutingGroup( + group_name="ordered-chat", + models=["primary", "backup"], + routing_strategy="priority", + model_priorities={"primary": 1, "backup": 2}, + ) + llm_router: Final = Router( + model_list=[ + { + "model_name": model, + "litellm_params": { + "model": "openai/gpt-5.4-nano", + "api_key": "sk-test", + }, + } + for model in group.models + ], + routing_groups=[group], + ) + expected: Final = [ + { + "group_name": "ordered-chat", + "models": ["primary", "backup"], + "routing_strategy": "priority", + "routing_strategy_args": None, + "model_priorities": ( + {"primary": 8, "backup": 3} + if from_config + else {"primary": 1, "backup": 2} + ), + } + ] + config: Final = {"routing_groups": expected} if from_config else {} + store: Final = SettingsStore("router_settings") + store.load_yaml(config) + monkeypatch.setattr(proxy_server, "llm_router", llm_router) + monkeypatch.setattr( + proxy_server, "proxy_config", _StubProxyConfig(store, config) + ) + admin_user: Final = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-test" + ) + + response: Final[RouterSettingsResponse] = await get_router_settings( + user_api_key_dict=admin_user + ) + + assert response.current_values["routing_groups"] == expected + groups_field: Final = next( + field for field in response.fields if field.field_name == "routing_groups" + ) + assert groups_field.field_value == expected diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py index acc7c8ca21a..b6eebcb2ef3 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py @@ -110,9 +110,7 @@ def patched_prisma(): @pytest.mark.asyncio -async def test_add_team_callbacks_rejects_unauthorized_caller( - patched_prisma, unauthorized_caller -): +async def test_add_team_callbacks_rejects_unauthorized_caller(patched_prisma, unauthorized_caller): data = AddTeamCallback( callback_name="langfuse", callback_type="success", @@ -133,9 +131,7 @@ async def test_add_team_callbacks_rejects_unauthorized_caller( @pytest.mark.asyncio -async def test_disable_team_logging_rejects_unauthorized_caller( - patched_prisma, unauthorized_caller -): +async def test_disable_team_logging_rejects_unauthorized_caller(patched_prisma, unauthorized_caller): with pytest.raises(HTTPException) as exc: await disable_team_logging( http_request=Mock(spec=Request), @@ -147,9 +143,7 @@ async def test_disable_team_logging_rejects_unauthorized_caller( @pytest.mark.asyncio -async def test_get_team_callbacks_rejects_unauthorized_caller( - patched_prisma, unauthorized_caller -): +async def test_get_team_callbacks_rejects_unauthorized_caller(patched_prisma, unauthorized_caller): with pytest.raises(HTTPException) as exc: await get_team_callbacks( http_request=Mock(spec=Request), @@ -180,9 +174,7 @@ async def test_proxy_admin_can_add_team_callbacks(patched_prisma): @pytest.mark.asyncio async def test_team_admin_of_target_team_can_add_callbacks(patched_prisma): - patched_prisma.get_data = AsyncMock( - return_value=_team_row(admin_user_id="team_admin_user") - ) + patched_prisma.get_data = AsyncMock(return_value=_team_row(admin_user_id="team_admin_user")) team_admin = UserAPIKeyAuth( user_role=LitellmUserRoles.INTERNAL_USER, @@ -472,9 +464,7 @@ async def test_add_team_callbacks_writes_encrypted_callback_vars(monkeypatch): litellm_changed_by=None, ) - written = json.loads( - mock_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"] - ) + written = json.loads(mock_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"]) cv = written["logging"][0]["callback_vars"] assert cv["langfuse_secret_key"] != "sk-lf-real-secret" assert cv["langfuse_public_key"] != "pk-lf-real-public" @@ -1192,8 +1182,12 @@ async def test_add_team_callbacks_rejects_team_deleted_before_write(): ) with ( - patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), # test-quality-ok: proxy_server module global is the endpoint's only injection point - patch("litellm.proxy.proxy_server.master_key", None), # test-quality-ok: proxy_server module global is the endpoint's only injection point + patch( + "litellm.proxy.proxy_server.prisma_client", mock_prisma + ), # test-quality-ok: proxy_server module global is the endpoint's only injection point + patch( + "litellm.proxy.proxy_server.master_key", None + ), # test-quality-ok: proxy_server module global is the endpoint's only injection point ): with pytest.raises(HTTPException) as exc: await add_team_callbacks( @@ -1482,12 +1476,16 @@ async def test_unknown_team_is_indistinguishable_from_no_access(call_handler, un a probe for valid team ids. The unknown-team response has to match the no-access one exactly, status and body. """ - with patch("litellm.proxy.proxy_server.prisma_client") as mock_client: # test-quality-ok: the handler imports prisma_client from proxy_server at call time, so there is no seam to inject through + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_client + ): # test-quality-ok: the handler imports prisma_client from proxy_server at call time, so there is no seam to inject through mock_client.get_data = AsyncMock(return_value=None) with pytest.raises(HTTPException) as unknown_team: await call_handler(unauthorized_caller) - with patch("litellm.proxy.proxy_server.prisma_client") as mock_client: # test-quality-ok: the handler imports prisma_client from proxy_server at call time, so there is no seam to inject through + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_client + ): # test-quality-ok: the handler imports prisma_client from proxy_server at call time, so there is no seam to inject through mock_client.get_data = AsyncMock(return_value=_team_row()) mock_client.db.litellm_teamtable.update = AsyncMock() with patch( # test-quality-ok: _verify_team_access calls this module-level helper directly, so there is no seam to inject through @@ -1508,7 +1506,9 @@ async def test_proxy_admin_still_told_the_team_is_unknown(): """The masking is only for callers who could not have managed the team; a proxy admin keeps the diagnosable error.""" admin = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin", api_key="sk-admin") - with patch("litellm.proxy.proxy_server.prisma_client") as mock_client: # test-quality-ok: the handler imports prisma_client from proxy_server at call time, so there is no seam to inject through + with ( + patch("litellm.proxy.proxy_server.prisma_client") as mock_client + ): # test-quality-ok: the handler imports prisma_client from proxy_server at call time, so there is no seam to inject through mock_client.get_data = AsyncMock(return_value=None) with pytest.raises(HTTPException) as exc: await get_team_callbacks( @@ -1526,13 +1526,29 @@ async def test_proxy_admin_still_told_the_team_is_unknown(): [ # the redirect, in every carrier a caller could pick: an entry naming # only a host, pairing with a key pair written on another entry - ({"langfuse_host": "http://attacker.invalid"}, [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], True), + ( + {"langfuse_host": "http://attacker.invalid"}, + [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], + True, + ), # the sibling carrier -- langfuse and langfuse_otel are one account - ({"langfuse_host": "http://attacker.invalid"}, [{"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_secret_key": "sk"}], True), + ( + {"langfuse_host": "http://attacker.invalid"}, + [{"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_secret_key": "sk"}], + True, + ), # a destination variable no integration registry lists ({"dd_agent_host": "attacker.invalid"}, [{"dd_api_key": "k", "dd_site": "us5.datadoghq.com"}], True), # one entry owning its family end to end is the feature - ({"langfuse_host": "https://eu.cloud.langfuse.com", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, [], False), + ( + { + "langfuse_host": "https://eu.cloud.langfuse.com", + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + }, + [], + False, + ), # a different family alongside an existing one stays fine ({"gcs_bucket_name": "bucket"}, [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], False), ({"langsmith_api_key": "k"}, [{"dd_api_key": "k"}], False), @@ -1541,19 +1557,55 @@ async def test_proxy_admin_still_told_the_team_is_unknown(): # the span scope picks what the family exports, not where to, so a second # entry may set either legal value next to the family's credentials ({"langfuse_span_scope": "llm_only"}, [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], False), - ({"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_span_scope": "full"}, [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_span_scope": "llm_only"}], False), + ( + {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_span_scope": "full"}, + [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_span_scope": "llm_only"}], + False, + ), # the scope on the stored entry must not shield a redirect riding next to it - ({"langfuse_host": "http://attacker.invalid", "langfuse_span_scope": "llm_only"}, [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_span_scope": "llm_only"}], True), + ( + {"langfuse_host": "http://attacker.invalid", "langfuse_span_scope": "llm_only"}, + [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_span_scope": "llm_only"}], + True, + ), # the same integration registered for a second event: identical values # flatten to the identical dict, so there is nothing to redirect - ({"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, [{"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], False), + ( + { + "langfuse_host": "https://us.cloud.langfuse.com", + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + }, + [ + { + "langfuse_host": "https://us.cloud.langfuse.com", + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + } + ], + False, + ), # the same credential under its other spelling is the same credential ({"langfuse_secret": "sk"}, [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], False), # a value the family already holds cannot be moved into another of its # variables either; the exporter would address or authenticate with it - ({"langfuse_host": "pk"}, [{"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_public_key": "pk"}], True), + ( + {"langfuse_host": "pk"}, + [{"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_public_key": "pk"}], + True, + ), # the same shape with one value moved is the redirect again - ({"langfuse_host": "http://attacker.invalid", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, [{"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], True), + ( + {"langfuse_host": "http://attacker.invalid", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, + [ + { + "langfuse_host": "https://us.cloud.langfuse.com", + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + } + ], + True, + ), ], ) def test_one_entry_owns_a_credential_family(new_vars, stored, rejected): @@ -1568,7 +1620,13 @@ def test_one_entry_owns_a_credential_family(new_vars, stored, rejected): @pytest.mark.asyncio -@pytest.mark.parametrize("caller", [_admin_auth(), UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="victim_admin", api_key="sk-team-admin")]) +@pytest.mark.parametrize( + "caller", + [ + _admin_auth(), + UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="victim_admin", api_key="sk-team-admin"), + ], +) async def test_a_second_entry_may_not_flip_the_span_scope(patched_prisma, caller): """The entries flatten last-wins at request time, so a failure entry saying llm_only next to a success entry saying full would export whichever is stored @@ -1580,7 +1638,11 @@ async def test_a_second_entry_may_not_flip_the_span_scope(patched_prisma, caller { "callback_name": "langfuse_otel", "callback_type": "success", - "callback_vars": {"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "langfuse_span_scope": "full"}, + "callback_vars": { + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "langfuse_span_scope": "full", + }, } ] } @@ -1610,3 +1672,35 @@ async def test_a_second_entry_may_not_flip_the_span_scope(patched_prisma, caller user_api_key_dict=caller, ) patched_prisma.db.litellm_teamtable.update.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_add_team_callbacks_rejects_out_of_range_arize_sampling_rate(patched_prisma): + data = AddTeamCallback( + callback_name="arize", + callback_type="success", + callback_vars={"arize_success_sampling_rate": "1.5"}, + ) + with pytest.raises(HTTPException) as exc: + await add_team_callbacks( + data=data, + http_request=Mock(spec=Request), + team_id="team-victim", + user_api_key_dict=_admin_auth(), + ) + assert exc.value.status_code == 400 + assert "arize_success_sampling_rate" in str(exc.value.detail) + patched_prisma.db.litellm_teamtable.update.assert_not_called() + + +def test_add_team_callback_accepts_arize_sampling_rate_vars(): + data = AddTeamCallback( + callback_name="arize", + callback_type="success", + callback_vars={ + "arize_success_sampling_rate": "0.5", + "arize_error_sampling_rate": "1.0", + }, + ) + assert data.callback_vars["arize_success_sampling_rate"] == "0.5" + assert data.callback_vars["arize_error_sampling_rate"] == "1.0" diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index e95359ace8a..7a9b6b66946 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -7,6 +7,7 @@ from collections.abc import Sequence from typing import Final, Optional, cast from unittest.mock import AsyncMock, MagicMock, PropertyMock, call, patch +import httpx import pytest from fastapi import HTTPException from fastapi.testclient import TestClient @@ -42,6 +43,8 @@ from litellm.proxy.management_endpoints.team_endpoints import ( _STRIP_DELETED_TEAM_FROM_USERS_SQL, GetTeamMemberPermissionsResponse, UpdateTeamMemberPermissionsRequest, + _build_team_list_where_conditions, + _get_org_admin_org_ids, _persist_deleted_team_records, _save_deleted_team_records, _transform_teams_to_deleted_records, @@ -5126,6 +5129,168 @@ async def test_team_member_delete_cleans_verification_tokens( ) +@pytest.mark.asyncio +async def test_team_member_delete_writes_deleted_audit_log_for_member_keys( + mock_db_client, mock_admin_auth +): + from litellm.proxy._types import ( + LiteLLM_VerificationToken, + LitellmTableNames, + TeamMemberDeleteRequest, + ) + from litellm.proxy.management_endpoints.team_endpoints import team_member_delete + + test_team_id = "team-del-audit-123" + test_user_id = "user-audit@example.com" + member_key = LiteLLM_VerificationToken(token="hashed-member-key", team_id=test_team_id, user_id=test_user_id) + + mock_team_row = MagicMock() + mock_team_row.model_dump.return_value = { + "team_id": test_team_id, + "members_with_roles": [ + {"user_id": test_user_id, "user_email": None, "role": "user"} + ], + "team_member_permissions": [], + "metadata": {}, + "models": [], + "spend": 0.0, + } + + mock_db_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=mock_team_row + ) + mock_db_client.db.litellm_teamtable.update = AsyncMock(return_value=mock_team_row) + + mock_user_row = MagicMock() + mock_user_row.user_id = test_user_id + mock_user_row.teams = [test_team_id] + mock_db_client.db.litellm_usertable.find_many = AsyncMock( + return_value=[mock_user_row] + ) + mock_db_client.db.litellm_usertable.update = AsyncMock(return_value=MagicMock()) + + mock_db_client.db.litellm_teammembership = MagicMock() + mock_db_client.db.litellm_teammembership.delete_many = AsyncMock( + return_value=MagicMock() + ) + + mock_db_client.db.litellm_verificationtoken = MagicMock() + mock_db_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[member_key]) + mock_db_client.db.litellm_verificationtoken.delete_many = AsyncMock( + return_value=MagicMock() + ) + mock_db_client.db.litellm_jwtkeymapping = MagicMock() + mock_db_client.db.litellm_jwtkeymapping.find_many = AsyncMock(return_value=[]) + mock_db_client.db.litellm_deletedverificationtoken = MagicMock() + mock_db_client.db.litellm_deletedverificationtoken.create_many = AsyncMock(return_value=MagicMock()) + + _wire_member_delete_tx(mock_db_client) + + captured: Final[list] = [] + + async def _capture(request_data): + captured.append(request_data) + + with ( + patch("litellm.store_audit_logs", True), + patch( + "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + new=_capture, + ), + ): + await team_member_delete( + data=TeamMemberDeleteRequest(team_id=test_team_id, user_id=test_user_id), + user_api_key_dict=mock_admin_auth, + ) + for _ in range(100): + if captured: + break + await asyncio.sleep(0.01) + + key_rows: Final = [r for r in captured if r.table_name == LitellmTableNames.KEY_TABLE_NAME] + assert len(key_rows) == 1 + audit_row: Final = key_rows[0] + assert audit_row.action == "deleted" + assert audit_row.object_id == member_key.token + assert audit_row.changed_by + assert json.loads(audit_row.before_value)["token"] == member_key.token + + +@pytest.mark.asyncio +async def test_delete_team_writes_deleted_audit_log_for_team_keys( + monkeypatch, +): + from litellm.proxy._types import DeleteTeamRequest, LiteLLM_VerificationToken, LitellmTableNames + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + team = LiteLLM_TeamTable( + team_id="team-doomed", + team_alias="doomed-team", + members_with_roles=[], + metadata={}, + model_max_budget={}, + model_spend={}, + ) + team_key = LiteLLM_VerificationToken(token="hashed-doomed-key", team_id="team-doomed") + + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=team) + mock_prisma_client.delete_data = AsyncMock(return_value={"deleted_teams": ["team-doomed"]}) + mock_prisma_client.db.litellm_deletedteamtable.create_many = AsyncMock() + mock_prisma_client.db.litellm_deletedverificationtoken.create_many = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[team_key]) + mock_prisma_client.db.execute_raw = AsyncMock() + mock_prisma_client.db.litellm_teammembership.delete_many = AsyncMock() + mock_prisma_client.get_data = AsyncMock(return_value=None) + + mock_tx = AsyncMock() + mock_tx.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + mock_tx_cm = MagicMock() + mock_tx_cm.__aenter__ = AsyncMock(return_value=mock_tx) + mock_tx_cm.__aexit__ = AsyncMock(return_value=False) + mock_prisma_client.db.tx = MagicMock(return_value=mock_tx_cm) + _wire_team_delete_tx(mock_prisma_client) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", UserApiKeyCache()) + monkeypatch.setattr("litellm.proxy.proxy_server.create_audit_log_for_update", AsyncMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") + + caller = UserAPIKeyAuth( + user_id="admin-user", + api_key="sk-admin", + user_role=LitellmUserRoles.PROXY_ADMIN.value, + ) + captured: Final[list] = [] + + async def _capture(request_data): + captured.append(request_data) + + monkeypatch.setattr("litellm.store_audit_logs", True) + monkeypatch.setattr( + "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + _capture, + ) + await delete_team( + data=DeleteTeamRequest(team_ids=["team-doomed"]), + http_request=MagicMock(), + user_api_key_dict=caller, + litellm_changed_by="admin-user", + ) + for _ in range(100): + if captured: + break + await asyncio.sleep(0.01) + + key_rows: Final = [r for r in captured if r.table_name == LitellmTableNames.KEY_TABLE_NAME] + assert len(key_rows) == 1 + audit_row: Final = key_rows[0] + assert audit_row.action == "deleted" + assert audit_row.object_id == team_key.token + assert audit_row.changed_by + assert json.loads(audit_row.before_value)["token"] == team_key.token + + @pytest.mark.asyncio async def test_team_member_delete_reads_on_the_lock_holding_transaction( mock_db_client, mock_admin_auth @@ -16569,3 +16734,81 @@ def test_team_member_update_request_rejects_unusable_temp_budget_increase(increa TeamMemberUpdateRequest( team_id="team-1", user_id="user-1", temp_budget_increase=increase, temp_budget_expiry="2030-01-01T00:00:00Z" ) + + +_DB_OUTAGE_503_BODY: Final = { + "error": { + "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", + "type": "no_db_connection", + "param": "None", + "code": "503", + } +} + + +def _user_read_raising(error: Exception) -> tuple[MagicMock, MagicMock]: + prisma_client = MagicMock() + prisma_client.db.litellm_usertable.find_unique = AsyncMock(side_effect=error) + cache = MagicMock() + cache.async_get_cache = AsyncMock(return_value=None) + cache.async_set_cache = AsyncMock() + return prisma_client, cache + + +def _db_unavailable_fallback_identity(route: str) -> UserAPIKeyAuth: + from litellm.proxy.auth.auth_exception_handler import DB_UNAVAILABLE_FALLBACK_USER_ID + + return UserAPIKeyAuth( + key_name="failed-to-connect-to-db", + token="failed-to-connect-to-db", + user_id=DB_UNAVAILABLE_FALLBACK_USER_ID, + user_role=LitellmUserRoles.INTERNAL_USER, + request_route=route, + ) + + +@pytest.mark.asyncio +async def test_get_org_admin_org_ids_propagates_a_db_outage_instead_of_answering_not_an_org_admin(): + prisma_client, cache = _user_read_raising(httpx.ConnectError("All connection attempts failed")) + + with patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True): + with pytest.raises(httpx.ConnectError): + await _get_org_admin_org_ids( + user_id="outage-probe-user", + prisma_client=prisma_client, + user_api_key_cache=cache, + proxy_logging_obj=None, + ) + + +@pytest.mark.asyncio +async def test_build_team_list_where_conditions_propagates_a_db_outage_instead_of_answering_user_not_found(): + prisma_client, cache = _user_read_raising(httpx.ConnectError("All connection attempts failed")) + + with patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True): + with pytest.raises(httpx.ConnectError): + await _build_team_list_where_conditions( + prisma_client=prisma_client, + team_id=None, + team_alias=None, + organization_id=None, + user_id="outage-probe-user", + use_deleted_table=False, + user_api_key_cache=cache, + proxy_logging_obj=None, + ) + + +def test_list_team_v2_answers_503_no_db_connection_when_the_callers_user_read_hits_a_db_outage(monkeypatch): + prisma_client, cache = _user_read_raising(httpx.ConnectError("All connection attempts failed")) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) + app.dependency_overrides[user_api_key_auth] = lambda: _db_unavailable_fallback_identity("/v2/team/list") + try: + with patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True): + response = TestClient(app, raise_server_exceptions=False).get("/v2/team/list") + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 503, response.text + assert response.json() == _DB_OUTAGE_503_BODY diff --git a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py index 3c05068c4b0..32e5bea613c 100644 --- a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py +++ b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py @@ -1,3 +1,4 @@ +import asyncio import copy import json from collections.abc import Callable, Mapping, Sequence @@ -7,7 +8,7 @@ from typing import Final import pytest from pydantic import BaseModel, ConfigDict, ValidationError -from litellm.proxy._types import LiteLLM_TeamTable, LitellmUserRoles, Member, UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_TeamTable, LitellmTableNames, LitellmUserRoles, Member, UserAPIKeyAuth from litellm.proxy.auth.auth_checks import jwt_key_mapping_cache_key from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.list_api.common import ManagementProblem @@ -748,3 +749,96 @@ def test_request_models_reject_unknown_fields(): BulkTeamMemberDeleteRequest.model_validate({"members": [{"user_id": "u1", "role": "admin"}]}) with pytest.raises(ValidationError, match="dry_run"): BulkDeleteUserRequest.model_validate({"user_ids": ["u1"], "dry_run": True}) + + +@pytest.mark.asyncio +async def test_bulk_delete_writes_deleted_audit_log_for_deleted_keys(mocker): + prisma = _FakePrisma( + users=[_user("u1", "t1"), _user("keep", "t1")], + teams=[_team("t1", "u1", "keep")], + tokens=[ + {"token": "team-key", "user_id": "u1", "team_id": "t1"}, + {"token": "personal-key", "user_id": "u1"}, + {"token": "keep-key", "user_id": "keep", "team_id": "t1"}, + ], + ) + + captured: Final[list] = [] + + async def _capture(request_data): + captured.append(request_data) + + mocker.patch("litellm.store_audit_logs", True) + mocker.patch( + "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + new=_capture, + ) + await _delete(prisma, ["u1"]) + for _ in range(100): + if len([r for r in captured if r.table_name == LitellmTableNames.KEY_TABLE_NAME]) >= 2: + break + await asyncio.sleep(0.01) + + key_rows: Final = [r for r in captured if r.table_name == LitellmTableNames.KEY_TABLE_NAME] + assert {r.object_id for r in key_rows} == {"team-key", "personal-key"} + assert {r.action for r in key_rows} == {"deleted"} + assert all(r.changed_by for r in key_rows) + assert {json.loads(r.before_value)["token"] for r in key_rows} == {"team-key", "personal-key"} + + +@pytest.mark.asyncio +async def test_bulk_member_delete_writes_deleted_audit_log_for_removed_team_keys(mocker): + prisma = _FakePrisma( + users=[_user("u1", "t1"), _user("keep", "t1")], + teams=[_team("t1", "u1", "keep")], + tokens=[ + {"token": "team-key", "user_id": "u1", "team_id": "t1"}, + {"token": "keep-key", "user_id": "keep", "team_id": "t1"}, + ], + ) + + captured: Final[list] = [] + + async def _capture(request_data): + captured.append(request_data) + + mocker.patch("litellm.store_audit_logs", True) + mocker.patch( + "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + new=_capture, + ) + await _remove(prisma, "t1", [{"user_id": "u1"}]) + for _ in range(100): + if captured: + break + await asyncio.sleep(0.01) + + key_rows: Final = [r for r in captured if r.table_name == LitellmTableNames.KEY_TABLE_NAME] + assert len(key_rows) == 1 + audit_row: Final = key_rows[0] + assert audit_row.action == "deleted" + assert audit_row.object_id == "team-key" + assert audit_row.changed_by + assert json.loads(audit_row.before_value)["token"] == "team-key" + + +@pytest.mark.asyncio +async def test_bulk_delete_skips_the_key_audit_log_when_the_tx_rolls_back(mocker): + prisma = _FakePrisma( + users=[_user("u1", "a-good", "z-bad"), _user("u2", "a-good")], + teams=[_team("a-good", "u1", "u2"), _team("z-bad", "u1")], + tokens=[{"token": "k1", "user_id": "u1", "team_id": "a-good"}], + fail_locks=frozenset({"z-bad"}), + ) + cache = _cache_with("k1") + + mocker.patch("litellm.store_audit_logs", True) + mock_audit_write = mocker.patch( + "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + new=mocker.AsyncMock(), + ) + results = await _delete(prisma, ["u1", "u2"], cache=cache) + + assert [(r.user_id, r.success) for r in results] == [("u1", False), ("u2", False)] + assert [t["token"] for t in prisma.db.litellm_verificationtoken.rows] == ["k1"] + mock_audit_write.assert_not_called() diff --git a/tests/test_litellm/proxy/middleware/test_budget_reservation_release_middleware.py b/tests/test_litellm/proxy/middleware/test_budget_reservation_release_middleware.py new file mode 100644 index 00000000000..f37a20dff8b --- /dev/null +++ b/tests/test_litellm/proxy/middleware/test_budget_reservation_release_middleware.py @@ -0,0 +1,349 @@ +""" +Tests for BudgetReservationReleaseMiddleware. + +Auth reserves budget before the handler runs and hands the reservation to the +request or socket state. A litellm call made through the async client wrapper +claims it for the cost callback that runs after the call; anything still unclaimed +when the response is done or the socket has closed would keep the spend counter +pinned until its TTL, so the middleware releases it. +""" + +import asyncio +from collections.abc import AsyncIterator, Awaitable, Callable, Mapping +from datetime import datetime +from typing import Final + +import pytest +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import JSONResponse, Response, StreamingResponse +from starlette.routing import Route +from starlette.types import ASGIApp, Message, Receive, Scope, Send +from starlette.websockets import WebSocket + +import litellm +from litellm.caching import DualCache +from litellm.proxy import proxy_server +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.middleware.budget_reservation_release_middleware import ( + BudgetReservationReleaseMiddleware, +) +from litellm.proxy.spend_tracking.budget_reservation import ( + reconcile_budget_reservation, + release_unbound_budget_reservation, + reserve_budget_for_request, +) +from litellm.proxy.utils import ProxyLogging +from litellm.utils import Rules, function_setup + +KEY_TOKEN: Final = "hashed-release-middleware-key" +COUNTER_KEY: Final = f"spend:key:{KEY_TOKEN}" +CHAT_BODY: Final = {"model": "gpt-4o", "messages": [{"role": "user", "content": "hello"}]} + + +@pytest.fixture +def spend_counter_cache(monkeypatch: pytest.MonkeyPatch) -> DualCache: + cache: Final = DualCache() + monkeypatch.setattr(proxy_server, "spend_counter_cache", cache) + monkeypatch.setattr(proxy_server, "prisma_client", None) + return cache + + +@pytest.fixture +def no_callbacks(monkeypatch: pytest.MonkeyPatch) -> None: + for callback_list_name in ( + "callbacks", + "success_callback", + "failure_callback", + "_async_success_callback", + "_async_failure_callback", + ): + monkeypatch.setattr(litellm, callback_list_name, []) + + +async def _reserve() -> dict: + reservation: Final = await reserve_budget_for_request( + request_body=CHAT_BODY, + route="/v1/chat/completions", + llm_router=None, + valid_token=UserAPIKeyAuth(token=KEY_TOKEN, max_budget=1.0, spend=0.0), + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=UserApiKeyCache(), + proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()), + ) + assert reservation is not None + assert reservation["reserved_cost"] > 0 + return reservation + + +async def _chat(reservation: dict, **kwargs: object) -> object: + return await litellm.acompletion( + **CHAT_BODY, + metadata={"user_api_key_budget_reservation": reservation}, + **kwargs, + ) + + +def _proxy_pre_call_setup(route_type: str, reservation: dict) -> None: + function_setup( + original_function=route_type, + rules_obj=Rules(), + start_time=datetime.now(), + **CHAT_BODY, + litellm_call_id="proxy-pre-call-setup", + metadata={"user_api_key_budget_reservation": reservation}, + ) + + +def _app( + handler: Callable[[Request], Awaitable[Response]], + release: Callable[[Mapping[str, object]], Awaitable[None]] = release_unbound_budget_reservation, +) -> Starlette: + app: Final = Starlette(routes=[Route("/", handler, methods=["POST"])]) + app.add_middleware(BudgetReservationReleaseMiddleware, release=release) + return app + + +async def _post(app: ASGIApp) -> None: + scope: Final = { + "type": "http", + "method": "POST", + "path": "/", + "raw_path": b"/", + "headers": [], + "query_string": b"", + "scheme": "http", + "server": ("testserver", 80), + "client": ("testclient", 1), + } + + body_delivered: Final = asyncio.Event() + client_never_disconnects: Final = asyncio.Event() + + async def receive() -> Message: + if body_delivered.is_set(): + await client_never_disconnects.wait() + body_delivered.set() + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(message: Message) -> None: + return None + + await app(scope, receive, send) + + +def _counter(spend_counter_cache: DualCache) -> float | None: + return spend_counter_cache.in_memory_cache.get_cache(key=COUNTER_KEY) + + +@pytest.mark.asyncio +async def test_unbound_reservation_is_released_after_the_response(spend_counter_cache: DualCache): + reservation: Final = await _reserve() + assert _counter(spend_counter_cache) == pytest.approx(reservation["reserved_cost"]) + + async def handler(request: Request) -> Response: + request.state.budget_reservation = reservation + return JSONResponse({"id": "batch_123", "status": "cancelling"}) + + await _post(_app(handler)) + + assert _counter(spend_counter_cache) == pytest.approx(0.0) + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_unbound_reservation_is_released_when_the_handler_raises(spend_counter_cache: DualCache): + reservation: Final = await _reserve() + + async def handler(request: Request) -> Response: + request.state.budget_reservation = reservation + raise RuntimeError("upstream refused the cancel") + + with pytest.raises(RuntimeError): + await _post(_app(handler)) + + assert _counter(spend_counter_cache) == pytest.approx(0.0) + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_reservation_seen_only_by_the_proxy_pre_call_logging_object_is_released( + spend_counter_cache: DualCache, no_callbacks: None +): + reservation: Final = await _reserve() + + async def cancel_batch_without_a_client_wrapper() -> dict: + return {"id": "batch_123", "status": "cancelling"} + + async def handler(request: Request) -> Response: + request.state.budget_reservation = reservation + _proxy_pre_call_setup("acancel_batch", reservation) + return JSONResponse(await cancel_batch_without_a_client_wrapper()) + + await _post(_app(handler)) + + assert _counter(spend_counter_cache) == pytest.approx(0.0) + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_reservation_of_a_failed_call_is_released_after_the_error_response( + spend_counter_cache: DualCache, no_callbacks: None +): + reservation: Final = await _reserve() + refused: Final = litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o") + + async def handler(request: Request) -> Response: + request.state.budget_reservation = reservation + _proxy_pre_call_setup("acompletion", reservation) + try: + await _chat(reservation, mock_response=refused) + except litellm.AuthenticationError: + return JSONResponse({"error": {"message": "bad key"}}, status_code=401) + raise AssertionError("the mocked call must fail") + + await _post(_app(handler)) + + assert _counter(spend_counter_cache) == pytest.approx(0.0) + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_reservation_claimed_by_a_completed_call_is_left_for_the_callback( + spend_counter_cache: DualCache, no_callbacks: None +): + reservation: Final = await _reserve() + reserved_cost: Final = reservation["reserved_cost"] + + async def handler(request: Request) -> Response: + request.state.budget_reservation = reservation + _proxy_pre_call_setup("acompletion", reservation) + response: Final = await _chat(reservation, mock_response="ok") + return JSONResponse(response.model_dump()) + + await _post(_app(handler)) + + assert _counter(spend_counter_cache) == pytest.approx(reserved_cost) + assert reservation["finalized"] is False + + +@pytest.mark.asyncio +async def test_reservation_claimed_by_a_streaming_call_is_left_for_the_callback_that_finishes_after_the_response( + spend_counter_cache: DualCache, no_callbacks: None +): + reservation: Final = await _reserve() + reserved_cost: Final = reservation["reserved_cost"] + + async def handler(request: Request) -> Response: + request.state.budget_reservation = reservation + _proxy_pre_call_setup("acompletion", reservation) + stream: Final = await _chat(reservation, mock_response="ok", stream=True) + + async def sse() -> AsyncIterator[bytes]: + async for chunk in stream: + yield f"data: {chunk.model_dump_json()}\n\n".encode() + yield b"data: [DONE]\n\n" + + return StreamingResponse(sse(), media_type="text/event-stream") + + await _post(_app(handler)) + + assert _counter(spend_counter_cache) == pytest.approx(reserved_cost) + assert reservation["finalized"] is False + + actual_cost: Final = reserved_cost / 4 + await reconcile_budget_reservation(budget_reservation=reservation, actual_cost=actual_cost) + + assert _counter(spend_counter_cache) == pytest.approx(actual_cost) + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_unbound_reservation_of_a_websocket_session_is_released_when_the_socket_closes( + spend_counter_cache: DualCache, +): + reservation: Final = await _reserve() + + async def listen_without_a_provider_key(scope: Scope, receive: Receive, send: Send) -> None: + websocket: Final = WebSocket(scope, receive, send) + websocket.state.budget_reservation = reservation + await websocket.close(code=1011, reason="Required 'DEEPGRAM_API_KEY' in environment") + + async def receive() -> Message: + return {"type": "websocket.connect"} + + async def send(message: Message) -> None: + return None + + middleware: Final = BudgetReservationReleaseMiddleware( + listen_without_a_provider_key, release=release_unbound_budget_reservation + ) + await middleware({"type": "websocket", "path": "/deepgram/v1/listen", "headers": []}, receive, send) + + assert _counter(spend_counter_cache) == pytest.approx(0.0) + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_release_runs_once_per_request_with_the_stamped_reservation(): + released: Final = [] + reservation: Final = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + + async def release(budget_reservation: Mapping[str, object]) -> None: + released.append(budget_reservation) + + async def handler(request: Request) -> Response: + request.state.budget_reservation = reservation + return JSONResponse({}) + + await _post(_app(handler, release=release)) + + assert released == [reservation] + assert released[0] is reservation + + +@pytest.mark.asyncio +async def test_request_without_a_reservation_releases_nothing(): + released: Final = [] + + async def release(budget_reservation: Mapping[str, object]) -> None: + released.append(budget_reservation) + + async def unauthenticated(request: Request) -> Response: + return JSONResponse({}) + + async def budget_checks_skipped(request: Request) -> Response: + request.state.budget_reservation = None + return JSONResponse({}) + + await _post(_app(unauthenticated, release=release)) + await _post(_app(budget_checks_skipped, release=release)) + + assert released == [] + + +@pytest.mark.asyncio +async def test_lifespan_scopes_pass_through(): + released: Final = [] + seen: Final = [] + + async def release(budget_reservation: Mapping[str, object]) -> None: + released.append(budget_reservation) + + async def inner(scope: Scope, receive: Receive, send: Send) -> None: + seen.append(scope["type"]) + + async def receive() -> Message: + return {"type": "lifespan.startup"} + + async def send(message: Message) -> None: + return None + + middleware: Final = BudgetReservationReleaseMiddleware(inner, release=release) + await middleware({"type": "lifespan", "state": {"budget_reservation": {"reserved_cost": 1.0}}}, receive, send) + + assert seen == ["lifespan"] + assert released == [] diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index ba8b5fa3ac4..094320225cd 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -588,43 +588,6 @@ class TestAzureAnthropicCostCalculation: == "claude-3-5-haiku-20241022" ) - def test_passthrough_logging_sets_response_cost_with_server_tool_use_dict(self): - from litellm.types.utils import Choices, Message, ModelResponse - - logging_obj = self._create_mock_logging_obj(model="claude-3-7-sonnet-20250219") - logging_obj.get_router_model_id.return_value = None - logging_obj.litellm_params = {} - - response = ModelResponse( - id="test-id", - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message(content="test", role="assistant"), - ) - ], - created=1234567890, - model="claude-3-7-sonnet-20250219", - usage={ - "prompt_tokens": 10, - "completion_tokens": 5, - "total_tokens": 15, - "server_tool_use": {"web_search_requests": 1}, - }, - ) - - kwargs = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( - litellm_model_response=response, - model="claude-3-7-sonnet-20250219", - kwargs={}, - start_time=datetime.now(), - end_time=datetime.now(), - logging_obj=logging_obj, - ) - - assert "response_cost" in kwargs - assert kwargs["response_cost"] > 0 class TestAnthropicBatchPassthroughCostTracking: @@ -2355,42 +2318,6 @@ class TestAnthropicResponseCostRecordedOnModelCallDetails: model_call_details["response_cost"], not from kwargs, so the streaming payload builder must record it there or streaming pass-through logs $0.""" - def test_create_payload_records_response_cost_on_model_call_details(self): - from litellm.types.utils import Choices, Message, ModelResponse - - logging_obj = MagicMock() - logging_obj.model_call_details = {} - logging_obj.get_router_model_id.return_value = None - logging_obj.litellm_params = {} - logging_obj.litellm_call_id = "test-call-id" - - response = ModelResponse( - id="test-id", - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message(content="hello", role="assistant"), - ) - ], - created=1234567890, - model="claude-3-7-sonnet-20250219", - usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, - ) - - kwargs = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( - litellm_model_response=response, - model="claude-3-7-sonnet-20250219", - kwargs={}, - start_time=datetime.now(), - end_time=datetime.now(), - logging_obj=logging_obj, - ) - - assert ( - logging_obj.model_call_details["response_cost"] == kwargs["response_cost"] - ) - assert logging_obj.model_call_details["response_cost"] > 0 class TestAnthropicPassthroughFastMode: diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_fal_ai_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_fal_ai_passthrough_logging_handler.py new file mode 100644 index 00000000000..1c945b9110b --- /dev/null +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_fal_ai_passthrough_logging_handler.py @@ -0,0 +1,132 @@ +"""Fal AI pass-through: upstream URL to model extraction and resolution-keyed spend tracking.""" + +from datetime import datetime +from typing import Final + +import pytest + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.fal_ai_passthrough_logging_handler import ( + FalAIPassthroughLoggingHandler, +) +from litellm.types.utils import ImageResponse + +pytestmark: Final = pytest.mark.usefixtures("local_model_cost_map") + +UPSTREAM_URL: Final = "https://queue.fal.run/fal-ai/trellis-2" + + +def _logging_obj(call_id: str = "call-fal") -> LiteLLMLoggingObj: + return LiteLLMLoggingObj( + model="unknown", + messages=[{"role": "user", "content": "passthrough"}], + stream=False, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id=call_id, + function_id="passthrough", + ) + + +def test_is_fal_ai_route_matches_only_the_fal_ai_provider(): + assert FalAIPassthroughLoggingHandler.is_fal_ai_route(UPSTREAM_URL, "fal_ai") is True + assert FalAIPassthroughLoggingHandler.is_fal_ai_route(UPSTREAM_URL, "deepgram") is False + assert FalAIPassthroughLoggingHandler.is_fal_ai_route(UPSTREAM_URL, None) is False + + +def test_handler_extracts_model_urls_and_resolution_keyed_cost(): + upstream_body: Final = { + "model_glb": {"url": "https://fal.media/model.glb", "content_type": "model/gltf-binary"}, + "images": [{"url": "https://fal.media/preview.png"}], + "timings": {"inference": 1.2}, + } + logging_obj: Final = _logging_obj() + expected_cost: Final = litellm.model_cost["fal_ai/fal-ai/trellis-2"]["output_cost_per_image_1536"] + + handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body=upstream_body, + request_body={"image_url": "https://example.com/in.png", "resolution": 1536}, + logging_obj=logging_obj, + url_route=UPSTREAM_URL, + kwargs={"litellm_params": {"metadata": {}}}, + ) + + result = handler_result["result"] + assert isinstance(result, ImageResponse) + assert [image.url for image in result.data or ()] == [ + "https://fal.media/model.glb", + "https://fal.media/preview.png", + ] + assert result._hidden_params["response_cost"] == pytest.approx(expected_cost) + assert handler_result["kwargs"]["model"] == "fal-ai/trellis-2" + assert handler_result["kwargs"]["custom_llm_provider"] == "fal_ai" + assert handler_result["kwargs"]["response_cost"] == pytest.approx(expected_cost) + assert handler_result["kwargs"]["litellm_params"] == {"metadata": {}} + assert logging_obj.model == "fal-ai/trellis-2" + assert logging_obj.model_call_details["model"] == "fal-ai/trellis-2" + assert logging_obj.model_call_details["custom_llm_provider"] == "fal_ai" + assert logging_obj.model_call_details["response_cost"] == pytest.approx(expected_cost) + + +def test_handler_charges_nothing_and_names_the_model_for_queue_status_and_result_polls(): + for upstream_url in ( + "https://queue.fal.run/fal-ai/trellis-2/requests/req-1/status", + "https://queue.fal.run/fal-ai/trellis-2/requests/req-1", + ): + logging_obj: Final = _logging_obj() + handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body={"status": "COMPLETED"}, + request_body={}, + logging_obj=logging_obj, + url_route=upstream_url, + kwargs={}, + ) + assert handler_result["kwargs"]["model"] == "fal-ai/trellis-2" + assert handler_result["kwargs"]["response_cost"] is None + assert logging_obj.model_call_details["response_cost"] is None + + +def test_handler_charges_for_queue_submit(): + handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body={"request_id": "req-1", "status": "IN_QUEUE"}, + request_body={"image_url": "https://example.com/in.png", "resolution": 1536}, + logging_obj=_logging_obj(), + url_route="https://queue.fal.run/fal-ai/trellis-2", + kwargs={}, + ) + assert handler_result["kwargs"]["model"] == "fal-ai/trellis-2" + assert handler_result["kwargs"]["response_cost"] == pytest.approx( + litellm.model_cost["fal_ai/fal-ai/trellis-2"]["output_cost_per_image_1536"] + ) + + +def test_handler_strips_queue_base_path_prefix(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("FAL_AI_QUEUE_API_BASE", "https://gw.example/fal/queue") + handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body={"request_id": "req-1", "status": "IN_QUEUE"}, + request_body={"image_url": "https://example.com/in.png", "resolution": 1536}, + logging_obj=_logging_obj(), + url_route="https://gw.example/fal/queue/fal-ai/trellis-2", + kwargs={}, + ) + + assert handler_result["kwargs"]["model"] == "fal-ai/trellis-2" + assert handler_result["kwargs"]["response_cost"] == pytest.approx( + litellm.model_cost["fal_ai/fal-ai/trellis-2"]["output_cost_per_image_1536"] + ) + + +def test_handler_without_url_values_returns_empty_image_response_and_no_cost(): + handler_result = FalAIPassthroughLoggingHandler().fal_ai_passthrough_handler( + response_body={"status": "COMPLETED"}, + request_body={}, + logging_obj=_logging_obj(), + url_route="https://queue.fal.run/fal-ai/no-such-model", + kwargs={}, + ) + + assert isinstance(handler_result["result"], ImageResponse) + assert not handler_result["result"].data + assert handler_result["kwargs"]["response_cost"] is None + assert handler_result["kwargs"]["model"] == "fal-ai/no-such-model" diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_tinyfish_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_tinyfish_passthrough_logging_handler.py new file mode 100644 index 00000000000..1b83c5140ca --- /dev/null +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_tinyfish_passthrough_logging_handler.py @@ -0,0 +1,400 @@ +import asyncio +import json +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest + +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.tinyfish_passthrough_logging_handler import ( + _BACKGROUND_BILLING_TASKS, + TinyFishPassthroughLoggingHandler, + is_tinyfish_agent_url, + resolve_tinyfish_agent_api_base, + resolve_tinyfish_cost_per_step, + run_id_from_sse_frames, + sse_poller_spawned, +) +from litellm.proxy.pass_through_endpoints.success_handler import ( + PassThroughEndpointLogging, +) +from litellm.types.passthrough_endpoints.tinyfish import is_allowed_tinyfish_endpoint + +RUN_URL = "https://agent.tinyfish.ai/v1/automation/run" +RUN_ASYNC_URL = "https://agent.tinyfish.ai/v1/automation/run-async" + + +def _make_logging_obj() -> MagicMock: + logging_obj = MagicMock() + logging_obj.litellm_call_id = "test-call-id" + logging_obj.model_call_details = {} + return logging_obj + + +def _make_response(method: str, url: str, body: dict) -> httpx.Response: + request = httpx.Request(method, url) + return httpx.Response(200, request=request, text=json.dumps(body)) + + +class _FakeClient: + """Payload items are dicts served with status_code, or (status, dict) tuples for scripted failures.""" + + def __init__(self, payloads: list, status_code: int = 200): + self.payloads = payloads + self.status_code = status_code + self.requested_urls: list[str] = [] + + async def get(self, url: str, headers: dict) -> httpx.Response: + self.requested_urls.append(url) + item = self.payloads[min(len(self.requested_urls) - 1, len(self.payloads) - 1)] + status, payload = item if isinstance(item, tuple) else (self.status_code, item) + return httpx.Response(status, text=json.dumps(payload), request=httpx.Request("GET", url)) + + +@pytest.fixture +def tinyfish_env(monkeypatch): + monkeypatch.setenv("TINYFISH_API_KEY", "sk-tf-test") + monkeypatch.delenv("TINYFISH_COST_PER_STEP", raising=False) + monkeypatch.delenv("TINYFISH_AGENT_API_BASE", raising=False) + + +class TestCostResolution: + def test_default_rate(self, tinyfish_env): + assert resolve_tinyfish_cost_per_step() == pytest.approx(0.016) + + def test_env_override(self, tinyfish_env, monkeypatch): + monkeypatch.setenv("TINYFISH_COST_PER_STEP", "0.02") + assert resolve_tinyfish_cost_per_step() == pytest.approx(0.02) + + def test_invalid_env_falls_back_to_default(self, tinyfish_env, monkeypatch): + monkeypatch.setenv("TINYFISH_COST_PER_STEP", "free") + assert resolve_tinyfish_cost_per_step() == pytest.approx(0.016) + + +class TestBillingGate: + @pytest.mark.parametrize( + "method,url,expected", + [ + ("POST", RUN_URL, True), + ("POST", RUN_ASYNC_URL, True), + ("POST", "https://agent.tinyfish.ai/v1/automation/run-sse", True), + ("GET", "https://agent.tinyfish.ai/v1/runs", False), + ("GET", "https://agent.tinyfish.ai/v1/runs/run-123?screenshots=none", False), + ("POST", "https://agent.tinyfish.ai/v1/runs/run-123/cancel", False), + ], + ) + def test_only_run_submissions_are_billed(self, method, url, expected): + assert TinyFishPassthroughLoggingHandler.should_log_request(method, url) is expected + + def test_polling_writes_no_spend_row(self, tinyfish_env): + logging_obj = _make_logging_obj() + logging_obj.dispatch_success_handlers = AsyncMock() + poll_url = "https://agent.tinyfish.ai/v1/runs/run-123" + + asyncio.run( + PassThroughEndpointLogging().pass_through_async_success_handler( + httpx_response=_make_response("GET", poll_url, {"run_id": "run-123", "status": "RUNNING"}), + response_body={"run_id": "run-123", "status": "RUNNING"}, + logging_obj=logging_obj, + url_route=poll_url, + result="", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={}, + passthrough_logging_payload={"url": poll_url}, + custom_llm_provider="tinyfish", + ) + ) + + logging_obj.dispatch_success_handlers.assert_not_awaited() + + +class TestBlockingRunBilling: + def _handle(self, response_body: dict, logging_obj: MagicMock): + return TinyFishPassthroughLoggingHandler.tinyfish_passthrough_handler( + httpx_response=_make_response("POST", RUN_URL, response_body), + response_body=response_body, + logging_obj=logging_obj, + url_route=RUN_URL, + result=json.dumps(response_body), + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"url": "https://scrapeme.live/shop", "goal": "extract products"}, + ) + + def test_bills_steps_times_rate(self, tinyfish_env): + logging_obj = _make_logging_obj() + run = {"run_id": "run-1", "status": "COMPLETED", "num_of_steps": 3, "result": {"products": []}} + + handler_result = self._handle(run, logging_obj) + + assert handler_result["kwargs"]["model"] == "tinyfish/automation-run" + assert handler_result["kwargs"]["custom_llm_provider"] == "tinyfish" + assert handler_result["kwargs"]["response_cost"] == pytest.approx(0.048) + assert "standard_logging_object" in handler_result["kwargs"] + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.048) + + def test_env_rate_override_applies(self, tinyfish_env, monkeypatch): + monkeypatch.setenv("TINYFISH_COST_PER_STEP", "0.5") + run = {"run_id": "run-1", "status": "COMPLETED", "num_of_steps": 2} + + handler_result = self._handle(run, _make_logging_obj()) + + assert handler_result["kwargs"]["response_cost"] == pytest.approx(1.0) + + def test_failed_run_logs_without_cost(self, tinyfish_env): + run = {"run_id": "run-1", "status": "FAILED", "num_of_steps": 2, "error": {"code": "AGENT_FAILURE"}} + + handler_result = self._handle(run, _make_logging_obj()) + + assert handler_result["kwargs"]["response_cost"] is None + + def test_cancelled_run_logs_without_cost(self, tinyfish_env): + run = {"run_id": "run-1", "status": "CANCELLED", "num_of_steps": 2} + + handler_result = self._handle(run, _make_logging_obj()) + + assert handler_result["kwargs"]["response_cost"] is None + + def test_null_steps_logs_without_cost(self, tinyfish_env): + run = {"run_id": "run-1", "status": "RUNNING", "num_of_steps": None} + + handler_result = self._handle(run, _make_logging_obj()) + + assert handler_result["kwargs"]["response_cost"] is None + + def test_unexpected_error_shape_still_bills(self, tinyfish_env): + run = {"run_id": "run-1", "status": "COMPLETED", "num_of_steps": 2, "error": {"retry_after": "5s"}} + + handler_result = self._handle(run, _make_logging_obj()) + + assert handler_result["kwargs"]["response_cost"] == pytest.approx(0.032) + + +class TestRunAsyncBilling: + def test_poll_and_log_bills_once_terminal(self, tinyfish_env): + logging_obj = _make_logging_obj() + logging_obj.dispatch_success_handlers = AsyncMock() + fake_client = _FakeClient( + payloads=[{"run_id": "run-9", "status": "COMPLETED", "num_of_steps": 4, "result": "ok"}] + ) + + asyncio.run( + TinyFishPassthroughLoggingHandler._poll_and_log( + run_id="run-9", + logging_obj=logging_obj, + result="", + start_time=datetime.now(), + cache_hit=False, + kwargs={}, + client=fake_client, + ) + ) + + logging_obj.dispatch_success_handlers.assert_awaited_once() + awaited_kwargs = logging_obj.dispatch_success_handlers.await_args.kwargs + assert awaited_kwargs["response_cost"] == pytest.approx(0.064) + assert awaited_kwargs["model"] == "tinyfish/automation-run" + assert fake_client.requested_urls == ["https://agent.tinyfish.ai/v1/runs/run-9?screenshots=none"] + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.064) + + def test_transient_poll_failure_keeps_polling(self, tinyfish_env): + fake_client = _FakeClient( + payloads=[(500, {}), {"run_id": "run-9", "status": "COMPLETED", "num_of_steps": 3}] + ) + + run = asyncio.run( + TinyFishPassthroughLoggingHandler._poll_until_terminal("run-9", fake_client, poll_interval_seconds=0.0) + ) + + assert run is not None + assert run["num_of_steps"] == 3 + assert len(fake_client.requested_urls) == 2 + + def test_gives_up_after_consecutive_poll_failures(self, tinyfish_env): + fake_client = _FakeClient(payloads=[(500, {})]) + + run = asyncio.run( + TinyFishPassthroughLoggingHandler._poll_until_terminal("run-9", fake_client, poll_interval_seconds=0.0) + ) + + assert run is None + assert len(fake_client.requested_urls) == 12 + + def test_traversal_run_id_is_rejected(self, tinyfish_env): + fake_client = _FakeClient(payloads=[{}]) + + run = asyncio.run(TinyFishPassthroughLoggingHandler._fetch_run("../vault/items", fake_client)) + + assert run is None + assert fake_client.requested_urls == [] + + def test_upstream_error_status_returns_none(self, tinyfish_env): + fake_client = _FakeClient(payloads=[{"error": {"code": "NOT_FOUND"}}], status_code=404) + + run = asyncio.run(TinyFishPassthroughLoggingHandler._fetch_run("run-1", fake_client)) + + assert run is None + + +class TestRunCostStatusGate: + def test_poller_bills_zero_for_terminal_failed_run(self, tinyfish_env): + logging_obj = _make_logging_obj() + logging_obj.dispatch_success_handlers = AsyncMock() + fake_client = _FakeClient(payloads=[{"run_id": "run-9", "status": "FAILED", "num_of_steps": 4}]) + + asyncio.run( + TinyFishPassthroughLoggingHandler._poll_and_log( + run_id="run-9", + logging_obj=logging_obj, + result="", + start_time=datetime.now(), + cache_hit=False, + kwargs={}, + client=fake_client, + ) + ) + + logging_obj.dispatch_success_handlers.assert_awaited_once() + assert logging_obj.dispatch_success_handlers.await_args.kwargs["response_cost"] is None + assert len(fake_client.requested_urls) == 1 + + +class TestRunIdFromSseFrames: + def test_finds_run_id_in_first_frame(self): + frames = b'data: {"run_id": "run-7", "event": "INITIALIZED"}\n\ndata: {"run_id": "run-7", "event": "ACTION"}\n\n' + assert run_id_from_sse_frames(frames) == "run-7" + + def test_skips_frames_without_run_id(self): + frames = b': keepalive\n\ndata: not-json\n\ndata: {"event": "HEARTBEAT"}\n\n' + assert run_id_from_sse_frames(frames) is None + + +class TestStartSseRunBilling: + def test_spawns_detached_poller_that_bills_once(self, tinyfish_env): + logging_obj = _make_logging_obj() + logging_obj.dispatch_success_handlers = AsyncMock() + logging_obj.model_call_details["litellm_params"] = {"metadata": {"user_api_key_hash": "hash-team-a"}} + fake_client = _FakeClient( + payloads=[{"run_id": "run-7", "status": "COMPLETED", "num_of_steps": 5, "result": "done"}] + ) + + async def _run() -> None: + tasks_before = set(_BACKGROUND_BILLING_TASKS) + TinyFishPassthroughLoggingHandler.start_sse_run_billing( + run_id="run-7", + litellm_logging_obj=logging_obj, + start_time=datetime.now(), + client=fake_client, + ) + assert sse_poller_spawned(logging_obj) + await asyncio.gather(*(_BACKGROUND_BILLING_TASKS - tasks_before)) + + asyncio.run(_run()) + + logging_obj.dispatch_success_handlers.assert_awaited_once() + awaited_kwargs = logging_obj.dispatch_success_handlers.await_args.kwargs + assert awaited_kwargs["response_cost"] == pytest.approx(0.08) + # a missing call id makes every poller row a NULL request_id primary-key collision + assert awaited_kwargs["standard_logging_object"]["id"] == "test-call-id" + # SLO consumers (Prometheus, Langfuse) must see the caller's attribution despite the empty poller kwargs + assert awaited_kwargs["standard_logging_object"]["metadata"]["user_api_key_hash"] == "hash-team-a" + assert fake_client.requested_urls == ["https://agent.tinyfish.ai/v1/runs/run-7?screenshots=none"] + + def test_flag_defaults_to_not_spawned(self): + assert not sse_poller_spawned(_make_logging_obj()) + def test_collected_chunks_price_via_run_fetch(self, tinyfish_env): + logging_obj = _make_logging_obj() + chunks = [ + 'data: {"type": "STARTED", "run_id": "run-7", "status": "RUNNING"}', + 'data: {"type": "PROGRESS", "run_id": "run-7"}', + 'data: {"type": "COMPLETE", "run_id": "run-7", "status": "COMPLETED", "result": "done"}', + ] + fake_client = _FakeClient( + payloads=[{"run_id": "run-7", "status": "COMPLETED", "num_of_steps": 5, "result": "done"}] + ) + + payload = asyncio.run( + TinyFishPassthroughLoggingHandler.handle_logging_tinyfish_collected_chunks( + litellm_logging_obj=logging_obj, + url_route="https://agent.tinyfish.ai/v1/automation/run-sse", + start_time=datetime.now(), + all_chunks=chunks, + end_time=datetime.now(), + client=fake_client, + ) + ) + + assert payload["kwargs"]["response_cost"] == pytest.approx(0.08) + assert payload["kwargs"]["model"] == "tinyfish/automation-run" + assert fake_client.requested_urls == ["https://agent.tinyfish.ai/v1/runs/run-7?screenshots=none"] + + def test_stream_without_run_id_logs_without_cost(self, tinyfish_env): + fake_client = _FakeClient(payloads=[{}]) + + payload = asyncio.run( + TinyFishPassthroughLoggingHandler.handle_logging_tinyfish_collected_chunks( + litellm_logging_obj=_make_logging_obj(), + url_route="https://agent.tinyfish.ai/v1/automation/run-sse", + start_time=datetime.now(), + all_chunks=["data: not-json", ": keepalive"], + end_time=datetime.now(), + client=fake_client, + ) + ) + + assert payload["kwargs"]["response_cost"] is None + assert fake_client.requested_urls == [] + + +class TestRouteDetection: + def test_provider_tag_claims_route(self): + assert PassThroughEndpointLogging().is_tinyfish_route("https://example.com/x", "tinyfish") + + def test_agent_host_claims_route(self): + assert PassThroughEndpointLogging().is_tinyfish_route("https://agent.tinyfish.ai/v1/runs", None) + + def test_other_providers_do_not_claim(self): + assert not PassThroughEndpointLogging().is_tinyfish_route("https://api.openai.com/v1", "openai") + + def test_env_base_override_claims_route(self, monkeypatch): + monkeypatch.setenv("TINYFISH_AGENT_API_BASE", "https://agent.staging.tinyfish.ai") + assert is_tinyfish_agent_url("https://agent.staging.tinyfish.ai/v1/runs/x") + assert not is_tinyfish_agent_url("https://agent.tinyfish.ai/v1/runs/x") + + def test_schemeless_env_base_is_normalized(self, monkeypatch): + monkeypatch.setenv("TINYFISH_AGENT_API_BASE", "agent.staging.tinyfish.ai") + assert resolve_tinyfish_agent_api_base() == "https://agent.staging.tinyfish.ai" + assert is_tinyfish_agent_url("https://agent.staging.tinyfish.ai/v1/runs/x") + + +class TestEndpointAllowlist: + @pytest.mark.parametrize( + "method,path,expected", + [ + ("POST", "/v1/automation/run", True), + ("POST", "/v1/automation/run-async", True), + ("POST", "/v1/automation/run-sse", True), + ("GET", "/v1/runs", False), + ("GET", "/v1/runs/run-abc-123", True), + ("POST", "/v1/runs/run-abc-123/cancel", True), + ("GET", "/v1/vault/items", False), + ("GET", "/v1/wallet", False), + ("POST", "/v1/browser-profiles", False), + ("DELETE", "/v1/runs/run-abc-123", False), + ("GET", "/v1/automation/run", False), + ("POST", "/v1/runs", False), + ("GET", "/v1/runs/..", False), + ("POST", "/v1/runs/../automation/run/cancel", False), + ("POST", "/v1/automation/run/", False), + ("POST", "/v1/automation/run-async/", False), + ("POST", "/v1/automation/run-sse/", False), + ("POST", "/v1//automation/run-async", False), + ("GET", "/v1/runs/run-abc-123/", False), + ("POST", "/v1/runs/run-abc-123/cancel/", False), + ], + ) + def test_allowlist(self, method, path, expected): + assert is_allowed_tinyfish_endpoint(method, path) is expected diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index 345eeeedc31..e0a5ef063e8 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -42,6 +42,7 @@ def _handler_result(response_body: dict, request_body: dict) -> dict: end_time=datetime.now(), cache_hit=False, request_body=request_body, + custom_llm_provider="typesafe", ) @@ -59,6 +60,7 @@ def test_uses_registry_pricing_and_standard_usage(): end_time=datetime.now(), cache_hit=False, request_body={"model": "jev-latest"}, + custom_llm_provider="typesafe", ) expected_cost = 312 * model_cost["input_cost_per_token"] + 48 * model_cost["output_cost_per_token"] @@ -105,6 +107,7 @@ def test_records_model_provider_and_cost_on_logging_details(): end_time=datetime.now(), cache_hit=False, request_body={"model": "jev-latest"}, + custom_llm_provider="typesafe", ) assert result["kwargs"]["model"] == "typesafe/jev-1.13.0" @@ -132,3 +135,76 @@ def test_success_handler_dispatches_to_typesafe_handler(): assert normalized["kwargs"]["custom_llm_provider"] == "typesafe" assert normalized["kwargs"]["model"] == "typesafe/jev-1.13.0" + + +def test_openrouter_decisions_response_is_priced_from_request_model_registry_row(): + logging_obj = _logging_obj() + model_cost = litellm.model_cost["openrouter/typesafe/jev-1.13"] + response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={ + "model": "typesafe/jev-1.13-20260917", + "usage": {"input_tokens": 282, "output_tokens": 20}, + }, + logging_obj=logging_obj, + url_route="https://openrouter.ai/api/alpha/decisions", + result='{"answers": {}}', + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"model": "typesafe/jev-1.13"}, + custom_llm_provider="openrouter", + ) + + expected_cost = 282 * model_cost["input_cost_per_token"] + 20 * model_cost["output_cost_per_token"] + assert response["kwargs"]["model"] == "openrouter/typesafe/jev-1.13-20260917" + assert response["kwargs"]["custom_llm_provider"] == "openrouter" + assert response["kwargs"]["response_cost"] == pytest.approx(expected_cost) + assert response["kwargs"]["combined_usage_object"].prompt_tokens == 282 + assert response["kwargs"]["combined_usage_object"].completion_tokens == 20 + assert response["kwargs"]["combined_usage_object"].total_tokens == 302 + + +def test_success_handler_dispatches_openrouter_to_the_shared_handler(): + logging_obj = _logging_obj() + normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=_response(), + response_body={ + "model": "typesafe/jev-1.13-20260917", + "usage": {"input_tokens": 282, "output_tokens": 20}, + }, + request_body={"model": "typesafe/jev-1.13"}, + logging_obj=logging_obj, + url_route="https://openrouter.ai/api/alpha/decisions", + result="{}", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + custom_llm_provider="openrouter", + ) + + assert normalized["kwargs"]["custom_llm_provider"] == "openrouter" + assert normalized["kwargs"]["model"] == "openrouter/typesafe/jev-1.13-20260917" + + +def test_success_handler_skips_typesafe_pricing_for_non_decisions_openrouter_routes(): + logging_obj = _logging_obj() + normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=_response(), + response_body={ + "model": "typesafe/jev-1.13-20260917", + "usage": {"input_tokens": 282, "output_tokens": 20}, + }, + request_body={"model": "typesafe/jev-1.13"}, + logging_obj=logging_obj, + url_route="https://openrouter.ai/api/v1/chat/completions", + result="{}", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + custom_llm_provider="openrouter", + ) + + assert normalized["standard_logging_response_object"] is None + assert "combined_usage_object" not in normalized["kwargs"] + assert normalized["kwargs"].get("model") != "openrouter/typesafe/jev-1.13-20260917" diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 636980eb6e3..353ffadfa46 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -29,6 +29,7 @@ from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( BaseOpenAIPassThroughHandler, RouteChecks, + _fal_target, _join_url_paths, _proxy_general_settings, anthropic_proxy_route, @@ -38,6 +39,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( bedrock_proxy_route, create_pass_through_route, cursor_proxy_route, + fal_ai_proxy_route, get_azure_ai_search_index_from_endpoint, get_vertex_base_url, is_azure_ai_search_service_level_index_create, @@ -47,6 +49,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( mistral_proxy_route, relay_nvidia_nim_request, openai_proxy_route, + openrouter_proxy_route, typesafe_proxy_route, vertex_discovery_proxy_route, vertex_proxy_route, @@ -7114,3 +7117,494 @@ class TestTypeSafePassthroughRoute: custom_llm_provider="typesafe", is_streaming_request=False, ) + + +class TestFalAIPassthroughRoute: + @pytest.fixture + def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + from litellm.proxy.proxy_server import app + + monkeypatch.setenv("FAL_AI_API_KEY", "fal-test-key") + monkeypatch.delenv("FAL_AI_QUEUE_API_BASE", raising=False) + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + yield TestClient(app) + + def test_submit_forwards_body_and_key_scheme_to_queue_fal_run(self, client: TestClient) -> None: + body: Final = {"image_url": "https://example.com/in.png", "resolution": 1536} + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post("https://queue.fal.run/fal-ai/trellis-2").mock( + return_value=httpx.Response(200, json={"request_id": "req-1", "status": "IN_QUEUE"}) + ) + response = client.post("/fal_ai/fal-ai/trellis-2", json=body) + + assert response.status_code == 200, response.text + assert response.json() == {"request_id": "req-1", "status": "IN_QUEUE"} + sent = route.calls.last.request + assert sent.headers["authorization"] == "Key fal-test-key" + assert json.loads(sent.content or b"{}") == body + + def test_status_get_forwards_to_queue_fal_run(self, client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + route = upstream.get("https://queue.fal.run/fal-ai/trellis-2/requests/req-1/status").mock( + return_value=httpx.Response(200, json={"status": "COMPLETED"}) + ) + response = client.get("/fal_ai/fal-ai/trellis-2/requests/req-1/status") + + assert response.status_code == 200, response.text + assert response.json() == {"status": "COMPLETED"} + assert route.calls.last.request.headers["authorization"] == "Key fal-test-key" + + def test_honours_fal_ai_queue_api_base_override(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FAL_AI_API_KEY", "fal-test-key") + monkeypatch.setenv("FAL_AI_QUEUE_API_BASE", "https://queue.example/base") + endpoint_func = AsyncMock(return_value={"ok": True}) + create_route = Mock(return_value=endpoint_func) + monkeypatch.setattr( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + create_route, + ) + request = MagicMock(spec=Request) + request.method = "POST" + request.query_params = {} + request.json = AsyncMock(return_value={}) + + result = asyncio.run( + fal_ai_proxy_route( + endpoint="fal-ai/trellis", + request=request, + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + ) + + assert result == {"ok": True} + create_route.assert_called_once_with( + endpoint="fal-ai/trellis", + target="https://queue.example/base/fal-ai/trellis", + custom_headers={"Authorization": "Key fal-test-key"}, + custom_llm_provider="fal_ai", + is_streaming_request=False, + ) + + def test_submit_to_unpriced_endpoint_returns_400_without_upstream_call(self, client: TestClient) -> None: + with respx.mock(assert_all_called=False) as upstream: + route = upstream.post("https://queue.fal.run/fal-ai/unpriced-model").mock( + return_value=httpx.Response(200, json={"request_id": "req-1"}) + ) + response = client.post("/fal_ai/fal-ai/unpriced-model", json={"image_url": "https://example.com/in.png"}) + + assert response.status_code == 400, response.text + assert "no pricing entry" in response.text + assert not route.calls + + def test_submit_to_catalog_key_the_pricer_cannot_price_returns_400_without_upstream_call( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setitem( + litellm.model_cost, + "fal_ai/fal-ai/priceless-model", + {"litellm_provider": "fal_ai", "mode": "image_generation"}, + ) + with respx.mock(assert_all_called=False) as upstream: + route = upstream.post("https://queue.fal.run/fal-ai/priceless-model").mock( + return_value=httpx.Response(200, json={"request_id": "req-1"}) + ) + response = client.post("/fal_ai/fal-ai/priceless-model", json={"image_url": "https://example.com/in.png"}) + + assert response.status_code == 400, response.text + assert "no pricing entry" in response.text + assert not route.calls + + def test_submit_gate_prices_the_request_body_not_an_empty_one( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setitem( + litellm.model_cost, + "fal_ai/fal-ai/keyed-only-model", + {"litellm_provider": "fal_ai", "mode": "image_generation", "output_cost_per_image_512": 0.02}, + ) + with respx.mock(assert_all_called=False) as upstream: + route = upstream.post("https://queue.fal.run/fal-ai/keyed-only-model").mock( + return_value=httpx.Response(200, json={"request_id": "req-1"}) + ) + priced = client.post( + "/fal_ai/fal-ai/keyed-only-model", json={"image_url": "https://example.com/in.png", "resolution": "512"} + ) + unpriced = client.post("/fal_ai/fal-ai/keyed-only-model", json={"image_url": "https://example.com/in.png"}) + + assert priced.status_code == 200, priced.text + assert unpriced.status_code == 400, unpriced.text + assert "no pricing entry" in unpriced.text + assert len(route.calls) == 1 + + def test_status_get_on_unpriced_endpoint_forwards(self, client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + upstream.get("https://queue.fal.run/fal-ai/unpriced-model/requests/req-9/status").mock( + return_value=httpx.Response(200, json={"status": "IN_PROGRESS"}) + ) + response = client.get("/fal_ai/fal-ai/unpriced-model/requests/req-9/status") + + assert response.status_code == 200, response.text + assert response.json() == {"status": "IN_PROGRESS"} + + def test_missing_fal_key_returns_401(self, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FAL_AI_API_KEY", raising=False) + response = client.post("/fal_ai/fal-ai/trellis", json={}) + assert response.status_code == 401 + + +class TestFalTargetSelection: + def test_endpoint_targets_queue_base(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FAL_AI_QUEUE_API_BASE", raising=False) + assert str(_fal_target("fal-ai/trellis-2")) == "https://queue.fal.run/fal-ai/trellis-2" + + def test_status_path_targets_queue_base(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("FAL_AI_QUEUE_API_BASE", raising=False) + assert str(_fal_target("fal-ai/trellis-2/requests/req-1/status")) == ( + "https://queue.fal.run/fal-ai/trellis-2/requests/req-1/status" + ) + + def test_queue_base_override(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FAL_AI_QUEUE_API_BASE", "https://queue.example/base") + assert str(_fal_target("fal-ai/trellis-2")) == "https://queue.example/base/fal-ai/trellis-2" + + +class TestOpenRouterPassthroughRoute: + @staticmethod + def _request(body: object, query_params: Mapping[str, str] | None = None) -> MagicMock: + request = MagicMock(spec=Request) + request.method = "POST" + request.query_params = query_params or {} + request.json = AsyncMock(return_value=body) + return request + + @pytest.fixture + def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + from litellm.proxy.proxy_server import app + + monkeypatch.setenv("OPENROUTER_API_KEY", "openrouter-test-key") + monkeypatch.setenv("OPENROUTER_API_BASE", "https://openrouter.example/base") + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + yield TestClient(app) + + @pytest.mark.parametrize( + "method, body", + [ + ("GET", None), + ("POST", {"state": "The sky is blue."}), + ("PUT", {"state": "The sky is blue."}), + ("DELETE", None), + ("PATCH", {"state": "The sky is blue."}), + ], + ) + def test_forwards_every_method_and_body_upstream( + self, client: TestClient, method: str, body: dict[str, str] | None + ) -> None: + with respx.mock(assert_all_called=True) as upstream: + route = upstream.request(method, "https://openrouter.example/base/alpha/decisions").mock( + return_value=httpx.Response(200, json={"id": "upstream_123"}) + ) + response = client.request(method, "/openrouter/alpha/decisions", json=body) + + assert (response.status_code, response.json()) == (200, {"id": "upstream_123"}) + sent: Final = route.calls.last.request + assert sent.headers["authorization"] == "Bearer openrouter-test-key" + assert json.loads(sent.content or b"{}") == (body or {}) + + @pytest.mark.asyncio + async def test_forwards_target_auth_provider_and_query(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENROUTER_API_KEY", "openrouter-test-key") + monkeypatch.setenv("OPENROUTER_API_BASE", "https://openrouter.example/base") + + async def fake_upstream(request, *_args): + target: Final = create_route.call_args.kwargs["target"] + upstream_url: Final = httpx.URL(target).copy_merge_params(request.query_params) + return {"upstream_query": parse_qs(upstream_url.query.decode())} + + endpoint_func = AsyncMock(side_effect=fake_upstream) + create_route = Mock(return_value=endpoint_func) + monkeypatch.setattr( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + create_route, + ) + + request = self._request({"state": "The sky is blue."}, {"trace": "yes"}) + result = await openrouter_proxy_route( + endpoint="alpha/decisions", + request=request, + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + + assert result == {"upstream_query": {"trace": ["yes"]}} + endpoint_func.assert_awaited_once() + create_route.assert_called_once_with( + endpoint="alpha/decisions", + target="https://openrouter.example/base/alpha/decisions", + custom_headers={ + "Authorization": "Bearer openrouter-test-key", + "Content-Type": "application/json", + }, + custom_llm_provider="openrouter", + is_streaming_request=False, + ) + + @pytest.mark.asyncio + async def test_uses_default_target_when_base_is_unset(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENROUTER_API_KEY", "openrouter-test-key") + monkeypatch.delenv("OPENROUTER_API_BASE", raising=False) + + endpoint_func = AsyncMock(return_value={"ok": True}) + create_route = Mock(return_value=endpoint_func) + monkeypatch.setattr( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + create_route, + ) + + await openrouter_proxy_route( + endpoint="alpha/decisions", + request=self._request({"state": "The sky is blue."}), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + + assert create_route.call_args.kwargs["target"] == "https://openrouter.ai/api/alpha/decisions" + + @pytest.mark.asyncio + @pytest.mark.parametrize("endpoint", ["alpha/decisions", "v1/chat/completions"]) + @pytest.mark.parametrize( + "base_env, expected_root", + [ + (None, "https://openrouter.ai/api"), + ("https://openrouter.ai/api/v1", "https://openrouter.ai/api"), + ("https://openrouter.example/base", "https://openrouter.example/base"), + ("https://openrouter.example/base/v1/", "https://openrouter.example/base"), + ], + ) + async def test_derives_api_root_from_configured_base( + self, monkeypatch: pytest.MonkeyPatch, base_env: str | None, expected_root: str, endpoint: str + ) -> None: + monkeypatch.setenv("OPENROUTER_API_KEY", "openrouter-test-key") + if base_env is None: + monkeypatch.delenv("OPENROUTER_API_BASE", raising=False) + else: + monkeypatch.setenv("OPENROUTER_API_BASE", base_env) + + endpoint_func = AsyncMock(return_value={"ok": True}) + create_route = Mock(return_value=endpoint_func) + monkeypatch.setattr( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + create_route, + ) + + await openrouter_proxy_route( + endpoint=endpoint, + request=self._request({"state": "The sky is blue."}), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + + assert create_route.call_args.kwargs["target"] == f"{expected_root}/{endpoint}" + + +class TestTinyFishProxyRoute: + """Tests for the TinyFish Agent pass-through route, faking the upstream HTTP boundary.""" + + RUN_BODY = {"url": "https://scrapeme.live/shop", "goal": "Extract the first 2 product names. Return JSON."} + + @pytest.fixture + def tinyfish_client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + from litellm.proxy.proxy_server import app + + monkeypatch.setenv("TINYFISH_API_KEY", "sk-tf-upstream") + monkeypatch.delenv("TINYFISH_AGENT_API_BASE", raising=False) + monkeypatch.delenv("TINYFISH_ALLOW_AUTHENTICATED_RUNS", raising=False) + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + yield TestClient(app) + + def test_forwards_run_with_server_key_not_callers(self, tinyfish_client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post("https://agent.tinyfish.ai/v1/automation/run").mock( + return_value=httpx.Response(200, json={"run_id": "run-1", "status": "COMPLETED", "num_of_steps": 2}) + ) + response = tinyfish_client.post( + "/tinyfish/v1/automation/run", json=self.RUN_BODY, headers={"X-API-Key": "sk-callers-virtual-key"} + ) + + assert (response.status_code, response.json()["run_id"]) == (200, "run-1") + assert route.calls.last.request.headers["x-api-key"] == "sk-tf-upstream" + + @pytest.mark.parametrize( + "method,path", + [ + ("GET", "/tinyfish/v1/vault/items"), + ("GET", "/tinyfish/v1/wallet"), + ("POST", "/tinyfish/v1/browser-profiles"), + ("GET", "/tinyfish/v1/automation/run"), + ("GET", "/tinyfish/v1/runs"), + ], + ) + def test_blocks_endpoints_outside_allowlist(self, tinyfish_client: TestClient, method: str, path: str) -> None: + with respx.mock: + response = tinyfish_client.request(method, path) + + assert response.status_code == 403 + assert "not an allowed TinyFish Agent passthrough endpoint" in response.json()["detail"] + + @pytest.mark.parametrize( + "path", + [ + "/tinyfish/v1/automation/run/", + "/tinyfish/v1/automation/run-async/", + "/tinyfish/v1/automation/run-sse/", + "/tinyfish/v1//automation/run-async", + ], + ) + def test_submit_paths_with_extra_slashes_are_rejected_before_forwarding( + self, tinyfish_client: TestClient, path: str + ) -> None: + with respx.mock(assert_all_called=False) as upstream: + upstream.post(url__regex=r"https://agent\.tinyfish\.ai/.*").mock( + return_value=httpx.Response(200, json={"run_id": "run-slash", "status": "PENDING"}) + ) + response = tinyfish_client.post(path, json=self.RUN_BODY) + + assert response.status_code == 403 + assert "not an allowed TinyFish Agent passthrough endpoint" in response.json()["detail"] + assert upstream.calls.call_count == 0 + + def test_rejects_authenticated_run_fields_by_default(self, tinyfish_client: TestClient) -> None: + with respx.mock: + response = tinyfish_client.post("/tinyfish/v1/automation/run", json={**self.RUN_BODY, "use_vault": True}) + + assert response.status_code == 403 + assert "use_vault" in response.json()["detail"] + + def test_env_opt_in_allows_authenticated_run_fields( + self, tinyfish_client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("TINYFISH_ALLOW_AUTHENTICATED_RUNS", "true") + + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post("https://agent.tinyfish.ai/v1/automation/run").mock( + return_value=httpx.Response(200, json={"run_id": "run-2", "status": "COMPLETED", "num_of_steps": 1}) + ) + response = tinyfish_client.post("/tinyfish/v1/automation/run", json={**self.RUN_BODY, "use_vault": True}) + + assert response.status_code == 200 + assert json.loads(route.calls.last.request.content)["use_vault"] is True + + def test_returns_401_on_missing_api_key( + self, tinyfish_client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("TINYFISH_API_KEY") + + with respx.mock: + response = tinyfish_client.get("/tinyfish/v1/runs/run-123") + + assert response.status_code == 401 + assert "TINYFISH_API_KEY" in response.json()["detail"] + + def test_env_base_override_changes_target( + self, tinyfish_client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("TINYFISH_AGENT_API_BASE", "https://agent.staging.tinyfish.ai") + + with respx.mock(assert_all_called=True) as upstream: + upstream.get("https://agent.staging.tinyfish.ai/v1/runs/run-123").mock( + return_value=httpx.Response(200, json={"run_id": "run-123", "status": "RUNNING"}) + ) + response = tinyfish_client.get("/tinyfish/v1/runs/run-123") + + assert (response.status_code, response.json()["status"]) == (200, "RUNNING") + + @pytest.mark.parametrize( + "body", + [ + {"custom_body": {"url": "https://scrapeme.live/shop", "goal": "g", "use_vault": True}}, + {"url": "https://scrapeme.live/shop", "goal": "g", "stream": True}, + {"url": "https://scrapeme.live/shop", "goal": "g", "query_params": {"x": "1"}}, + ], + ) + def test_rejects_passthrough_envelope_controls(self, tinyfish_client: TestClient, body: dict) -> None: + """custom_body smuggled vault fields past the 403 gate and a stream flag flipped the + billing mode, because the generic passthrough honors both from the caller's body.""" + with respx.mock as upstream: + route = upstream.post("https://agent.tinyfish.ai/v1/automation/run").mock( + return_value=httpx.Response(200, json={"run_id": "run-1", "status": "COMPLETED", "num_of_steps": 1}) + ) + response = tinyfish_client.post("/tinyfish/v1/automation/run", json=body) + + assert response.status_code == 400 + assert "envelope" in response.json()["detail"] + assert not route.called + + def test_rejects_envelope_stream_on_cancel(self, tinyfish_client: TestClient) -> None: + with respx.mock as upstream: + route = upstream.post("https://agent.tinyfish.ai/v1/runs/run-1/cancel").mock( + return_value=httpx.Response(200, json={"run_id": "run-1", "status": "CANCELLED"}) + ) + response = tinyfish_client.post("/tinyfish/v1/runs/run-1/cancel", json={"stream": True}) + + assert response.status_code == 400 + assert not route.called + + @pytest.mark.parametrize( + "content,content_type", + [ + ("url=https%3A%2F%2Fscrapeme.live%2Fshop&goal=g&stream=true", "application/x-www-form-urlencoded"), + ("url=https%3A%2F%2Fscrapeme.live%2Fshop&goal=g&use_vault=true", "application/x-www-form-urlencoded"), + ('{"url": "https://scrapeme.live/shop", "goal": "g", "use_vault": true}', "text/plain"), + ('[{"url": "https://scrapeme.live/shop", "goal": "g", "stream": true}]', "application/json"), + ], + ) + def test_rejects_bodies_that_are_not_json_objects( + self, tinyfish_client: TestClient, content: str, content_type: str + ) -> None: + """A form-encoded body carried stream and use_vault past both field gates, because + the gates only saw fields the body parsed to as JSON.""" + with respx.mock as upstream: + route = upstream.post("https://agent.tinyfish.ai/v1/automation/run").mock( + return_value=httpx.Response(200, json={"run_id": "run-1", "status": "COMPLETED", "num_of_steps": 1}) + ) + response = tinyfish_client.post( + "/tinyfish/v1/automation/run", content=content, headers={"Content-Type": content_type} + ) + + assert response.status_code == 400 + assert "JSON object" in response.json()["detail"] + assert not route.called + + def test_cancel_without_body_forwards(self, tinyfish_client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + upstream.post("https://agent.tinyfish.ai/v1/runs/run-1/cancel").mock( + return_value=httpx.Response(200, json={"run_id": "run-1", "status": "CANCELLED"}) + ) + response = tinyfish_client.post("/tinyfish/v1/runs/run-1/cancel") + + assert (response.status_code, response.json()["status"]) == (200, "CANCELLED") + + +class TestTinyFishRouteTimeout: + def test_default_covers_upstream_run_cap(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import _tinyfish_route_timeout + + monkeypatch.setattr(proxy_server, "general_settings", {}, raising=False) + assert _tinyfish_route_timeout() == 1500.0 + + def test_operator_configured_timeout_wins(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import _tinyfish_route_timeout + + monkeypatch.setattr(proxy_server, "general_settings", {"pass_through_request_timeout": 30}, raising=False) + assert _tinyfish_route_timeout() is None diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index fb89e3a6973..6ad850866b7 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -4256,6 +4256,112 @@ async def test_pass_through_request_non_streaming_success_unchanged(): mock_success_handler.assert_called_once() +@pytest.mark.asyncio +@pytest.mark.parametrize( + "upstream_status_code, claimed_by_the_success_handler", + [(200, True), (500, False)], + ids=["success-claims-the-reservation", "upstream-error-leaves-it-for-the-request-end-release"], +) +async def test_pass_through_request_claims_the_budget_reservation_only_when_its_success_handler_runs( + upstream_status_code: int, claimed_by_the_success_handler: bool +): + reservation: Final = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + user_api_key_dict: Final = UserAPIKeyAuth(api_key="hashed") + user_api_key_dict.budget_reservation = reservation + upstream_response: Final = httpx.Response( + status_code=upstream_status_code, + headers={"content-type": "application/json"}, + content=b'{"status": "upstream"}', + request=httpx.Request("POST", "http://target-api.com/api/generate"), + ) + + with ( + patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging, + patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client") as mock_get_client, + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.ProxyBaseLLMRequestProcessing" + ) as mock_processing, + patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.GLOBAL_LOGGING_WORKER") as mock_worker, + ): + mock_proxy_logging.pre_call_hook = AsyncMock(return_value={}) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value=None) + mock_processing.get_custom_headers.return_value = {} + mock_worker.ensure_initialized_and_enqueue = MagicMock(side_effect=lambda async_coroutine: async_coroutine.close()) + async_client = MagicMock() + async_client.build_request = MagicMock(return_value=MagicMock()) + async_client.send = AsyncMock(return_value=upstream_response) + mock_get_client.return_value = MagicMock(client=async_client) + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.url = "http://test-proxy.com/mock-upstream/api/generate" + mock_request.body = AsyncMock(return_value=b'{"prompt": "hi"}') + mock_request.headers = Headers({"content-type": "application/json"}) + mock_request.query_params = QueryParams({}) + + response = await pass_through_request( + request=mock_request, + target="http://target-api.com/api/generate", + custom_headers={}, + user_api_key_dict=user_api_key_dict, + ) + + assert response.status_code == upstream_status_code + assert reservation["callback_bound"] is claimed_by_the_success_handler + assert mock_worker.ensure_initialized_and_enqueue.call_count == int(claimed_by_the_success_handler) + + +@pytest.mark.asyncio +async def test_pass_through_request_leaves_the_budget_reservation_for_the_request_end_release_when_its_success_handler_cannot_be_enqueued(): + reservation: Final = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + user_api_key_dict: Final = UserAPIKeyAuth(api_key="hashed") + user_api_key_dict.budget_reservation = reservation + upstream_response: Final = httpx.Response( + status_code=200, + headers={"content-type": "application/json"}, + content=b'{"status": "upstream"}', + request=httpx.Request("POST", "http://target-api.com/api/generate"), + ) + + def refuse_to_enqueue(async_coroutine): + async_coroutine.close() + raise RuntimeError("logging worker is shutting down") + + with ( + patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging, + patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client") as mock_get_client, + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.ProxyBaseLLMRequestProcessing" + ) as mock_processing, + patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.GLOBAL_LOGGING_WORKER") as mock_worker, + ): + mock_proxy_logging.pre_call_hook = AsyncMock(return_value={}) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value=None) + mock_processing.get_custom_headers.return_value = {} + mock_worker.ensure_initialized_and_enqueue = MagicMock(side_effect=refuse_to_enqueue) + async_client = MagicMock() + async_client.build_request = MagicMock(return_value=MagicMock()) + async_client.send = AsyncMock(return_value=upstream_response) + mock_get_client.return_value = MagicMock(client=async_client) + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.url = "http://test-proxy.com/mock-upstream/api/generate" + mock_request.body = AsyncMock(return_value=b'{"prompt": "hi"}') + mock_request.headers = Headers({"content-type": "application/json"}) + mock_request.query_params = QueryParams({}) + + with pytest.raises(ProxyException): + await pass_through_request( + request=mock_request, + target="http://target-api.com/api/generate", + custom_headers={}, + user_api_key_dict=user_api_key_dict, + ) + + assert reservation["callback_bound"] is False + + @pytest.mark.asyncio async def test_pass_through_request_internal_failure_still_raises_proxy_exception(): """ @@ -4651,6 +4757,90 @@ async def test_pass_through_request_upstream_error_body_stays_buffered(): await fake_client.aclose() +_UPSTREAM_JSON_ERROR: Final = b'{"error": {"message": "bad request", "type": "invalid_request_error"}}' + + +async def _relay_upstream_through_pass_through_request( + general_settings, status_code, content_type, body, callback_headers=None +): + from litellm.proxy._types import UserAPIKeyAuth + + fake_client, cleanup = _inject_fake_passthrough_client( + _FakeUpstreamTransport( + status_code=status_code, + headers={"content-type": content_type}, + stream=_RecordingUpstreamByteStream((body,)), + ), + timeout=313.0, + ) + try: + with ExitStack() as stack: + mock_proxy_logging, _ = _enter_relay_logging_mocks(stack, {}) + mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value=callback_headers) + stack.enter_context(patch("litellm.proxy.proxy_server.general_settings", general_settings)) + return await pass_through_request( + request=_relay_client_request(), + target="http://upstream.test/v1/messages", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"), + timeout=313.0, + ) + finally: + cleanup() + await fake_client.aclose() + + +@pytest.mark.asyncio +async def test_pass_through_error_body_carries_the_call_id_when_opted_in(): + """With include_call_id_in_error_body on, a buffered upstream JSON error gets a top-level + litellm_call_id byte-identical to the x-litellm-call-id header, and content-length still + matches the rewritten body.""" + response = await _relay_upstream_through_pass_through_request( + {"include_call_id_in_error_body": True}, 400, "application/json", _UPSTREAM_JSON_ERROR + ) + + call_id = response.headers["x-litellm-call-id"] + assert response.status_code == 400 + assert json.loads(response.body) == {**json.loads(_UPSTREAM_JSON_ERROR), "litellm_call_id": call_id} + assert int(response.headers["content-length"]) == len(response.body) + + +@pytest.mark.asyncio +async def test_pass_through_error_body_call_id_follows_a_restamped_header(): + """A post_call_response_headers_hook that rewrites x-litellm-call-id wins in the header, so the + body copies the emitted header value rather than the id the proxy generated.""" + response = await _relay_upstream_through_pass_through_request( + {"include_call_id_in_error_body": True}, + 400, + "application/json", + _UPSTREAM_JSON_ERROR, + callback_headers={"x-litellm-call-id": "restamped-by-hook"}, + ) + + assert response.headers["x-litellm-call-id"] == "restamped-by-hook" + assert json.loads(response.body)["litellm_call_id"] == "restamped-by-hook" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "general_settings, status_code, content_type, body", + [ + ({}, 400, "application/json", _UPSTREAM_JSON_ERROR), + ({"include_call_id_in_error_body": True}, 502, "text/plain", b"upstream exploded"), + ({"include_call_id_in_error_body": True}, 200, "application/json", b'{"id": "msg_1", "type": "message"}'), + ], +) +async def test_pass_through_body_stays_byte_identical_outside_the_opt_in( + general_settings, status_code, content_type, body +): + """Opted out, a non-JSON error, or a success body: the upstream bytes are relayed as-is.""" + response = await _relay_upstream_through_pass_through_request(general_settings, status_code, content_type, body) + + assert response.status_code == status_code + assert response.body == body + assert "x-litellm-call-id" in response.headers + + _PARTIAL_RELAY_WARNING_MARKER = "ended before upstream body was fully relayed" diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py b/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py index ea6adc35b9a..00a3606bbf5 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py @@ -7,8 +7,14 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +import respx import litellm +import litellm.proxy.pass_through_endpoints.llm_provider_handlers.tinyfish_passthrough_logging_handler as tinyfish_handler_module +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.tinyfish_passthrough_logging_handler import ( + mark_sse_poller_spawned, + sse_poller_spawned, +) from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -838,3 +844,286 @@ async def test_chunk_processor_bills_partial_google_usage_on_mid_stream_exceptio assert failure_payload["completion_tokens"] == 12 assert failure_payload["response_cost"] > 12 * 3.75e-06 assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadTimeout) + + +class TestTinyFishStreamBilling: + """SSE billing is owned by the detached poller spawned on the first run_id frame; it must + survive gen.aclose() (client disconnect) and the stream-end path must not double-bill.""" + + RUNS_URL = "https://agent.tinyfish.ai/v1/runs/run-sse-1?screenshots=none" + SSE_ROUTE = "https://agent.tinyfish.ai/v1/automation/run-sse" + + @pytest.fixture + def tinyfish_env(self, monkeypatch): + monkeypatch.setenv("TINYFISH_API_KEY", "sk-tf-test") + monkeypatch.delenv("TINYFISH_COST_PER_STEP", raising=False) + monkeypatch.delenv("TINYFISH_AGENT_API_BASE", raising=False) + # aiohttp transport bypasses respx; force plain httpx and drop any cached aiohttp client + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + def _tinyfish_logging_obj(self): + obj = _unarmed_logging_obj() + obj.model_call_details = {} + obj.dispatch_success_handlers = AsyncMock() + return obj + + def _spawned_since(self, tasks_before): + return tinyfish_handler_module._BACKGROUND_BILLING_TASKS - tasks_before + + @pytest.mark.asyncio + async def test_run_id_split_across_chunks_spawns_one_poller(self, tinyfish_env): + chunks = [ + b'data: {"run_id": "run-s', + b'se-1", "event": "INITIALIZED"}\n\n', + b'data: {"run_id": "run-sse-1", "event": "COMPLETE"}\n\n', + ] + logging_obj = self._tinyfish_logging_obj() + tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS) + + with respx.mock(assert_all_called=True) as upstream: + upstream.get(self.RUNS_URL).respond( + json={"run_id": "run-sse-1", "status": "COMPLETED", "num_of_steps": 2, "result": "ok"} + ) + received = [] + async for chunk in PassThroughStreamingHandler.chunk_processor( + response=_make_streaming_response(chunks), + request_body={"url": "https://scrapeme.live/shop", "goal": "extract"}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.TINYFISH, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route=self.SSE_ROUTE, + route_streaming_logging=AsyncMock(), + ): + received.append(chunk) + + assert received == chunks + assert sse_poller_spawned(logging_obj) + spawned = self._spawned_since(tasks_before) + assert len(spawned) == 1 + await asyncio.gather(*spawned) + + logging_obj.dispatch_success_handlers.assert_awaited_once() + assert logging_obj.dispatch_success_handlers.await_args.kwargs["response_cost"] == pytest.approx(0.032) + + @pytest.mark.asyncio + async def test_poller_survives_client_disconnect_and_bills(self, tinyfish_env): + chunks = [ + b'data: {"run_id": "run-sse-1", "event": "INITIALIZED"}\n\n', + b'data: {"run_id": "run-sse-1", "event": "ACTION"}\n\n', + ] + logging_obj = self._tinyfish_logging_obj() + tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS) + + with respx.mock(assert_all_called=True) as upstream: + upstream.get(self.RUNS_URL).respond( + json={"run_id": "run-sse-1", "status": "COMPLETED", "num_of_steps": 2, "result": "ok"} + ) + gen = PassThroughStreamingHandler.chunk_processor( + response=_make_streaming_response(chunks), + request_body={"url": "https://scrapeme.live/shop", "goal": "extract"}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.TINYFISH, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route=self.SSE_ROUTE, + route_streaming_logging=AsyncMock(), + ) + await gen.__anext__() + spawned = self._spawned_since(tasks_before) + assert len(spawned) == 1 + await gen.aclose() + + task = next(iter(spawned)) + assert not task.cancelled() + await task + + logging_obj.dispatch_success_handlers.assert_awaited_once() + assert logging_obj.dispatch_success_handlers.await_args.kwargs["response_cost"] == pytest.approx(0.032) + + @pytest.mark.asyncio + async def test_stream_without_run_id_spawns_nothing(self, tinyfish_env): + logging_obj = self._tinyfish_logging_obj() + tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS) + + async for _ in PassThroughStreamingHandler.chunk_processor( + response=_make_streaming_response([b": keepalive\n\n", b'data: {"event": "HEARTBEAT"}\n\n']), + request_body={}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.TINYFISH, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route=self.SSE_ROUTE, + route_streaming_logging=AsyncMock(), + ): + pass + + assert not sse_poller_spawned(logging_obj) + assert self._spawned_since(tasks_before) == set() + + @pytest.mark.asyncio + async def test_stream_end_skips_dispatch_when_poller_owns_billing(self, tinyfish_env): + logging_obj = self._tinyfish_logging_obj() + mark_sse_poller_spawned(logging_obj) + + await PassThroughStreamingHandler._route_streaming_logging_to_handler( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route=self.SSE_ROUTE, + request_body={}, + endpoint_type=EndpointType.TINYFISH, + start_time=datetime.now(), + raw_bytes=[b'data: {"run_id": "run-sse-1", "event": "COMPLETE"}\n\n'], + end_time=datetime.now(), + ) + + logging_obj.dispatch_success_handlers.assert_not_awaited() + + @pytest.mark.asyncio + async def test_stream_end_fallback_still_logs_when_no_poller_spawned(self, tinyfish_env): + logging_obj = self._tinyfish_logging_obj() + + await PassThroughStreamingHandler._route_streaming_logging_to_handler( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route=self.SSE_ROUTE, + request_body={}, + endpoint_type=EndpointType.TINYFISH, + start_time=datetime.now(), + raw_bytes=[b": keepalive\n\n"], + end_time=datetime.now(), + ) + + logging_obj.dispatch_success_handlers.assert_awaited_once() + assert logging_obj.dispatch_success_handlers.await_args.kwargs["response_cost"] is None + + @pytest.mark.asyncio + async def test_upstream_error_after_spawn_skips_failure_dispatch(self, tinyfish_env): + logging_obj = self._tinyfish_logging_obj() + logging_obj.dispatch_failure_handlers = MagicMock() + tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS) + + async def _aiter_bytes(): + yield b'data: {"run_id": "run-sse-1", "event": "INITIALIZED"}\n\n' + raise httpx.ReadTimeout("upstream died") + + response = MagicMock(spec=httpx.Response) + response.status_code = 200 + response.aiter_bytes = _aiter_bytes + + async def _consume(): + async for _ in PassThroughStreamingHandler.chunk_processor( + response=response, + request_body={}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.TINYFISH, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route=self.SSE_ROUTE, + route_streaming_logging=AsyncMock(), + ): + pass + + with pytest.raises(httpx.ReadTimeout): + await _consume() + + spawned = self._spawned_since(tasks_before) + assert len(spawned) == 1 + # the poller owns the single row; a failure dispatch would collide on its request_id + logging_obj.dispatch_failure_handlers.assert_not_called() + for task in spawned: + task.cancel() + + @pytest.mark.asyncio + async def test_unterminated_run_id_frame_late_spawns_poller(self, tinyfish_env): + logging_obj = self._tinyfish_logging_obj() + tasks_before = set(tinyfish_handler_module._BACKGROUND_BILLING_TASKS) + + await PassThroughStreamingHandler._route_streaming_logging_to_handler( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route=self.SSE_ROUTE, + request_body={}, + endpoint_type=EndpointType.TINYFISH, + start_time=datetime.now(), + raw_bytes=[b'data: {"run_id": "run-sse-1", "event": "INITIALIZED"}'], + end_time=datetime.now(), + ) + + spawned = self._spawned_since(tasks_before) + assert len(spawned) == 1 + assert sse_poller_spawned(logging_obj) + # the poller polls to terminal instead of the old single fetch that mispriced a RUNNING run at $0 + logging_obj.dispatch_success_handlers.assert_not_awaited() + for task in spawned: + task.cancel() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "deferred_dispatch_armed", + [False, True], + ids=["enqueued-at-end-of-stream", "parked-for-deferred-dispatch"], +) +async def test_chunk_processor_claims_the_budget_reservation_before_handing_it_to_the_cost_callback( + deferred_dispatch_armed: bool, +): + reservation = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + response = _make_streaming_response([b"event-1", b"event-2"]) + logging_obj = _unarmed_logging_obj() + logging_obj.litellm_params = {"metadata": {"user_api_key_budget_reservation": reservation}} + if deferred_dispatch_armed: + logging_obj._on_deferred_stream_complete = AsyncMock() + claimed_when_the_callback_ran = [] + + async def cost_callback(**kwargs): + claimed_when_the_callback_ran.append(reservation["callback_bound"]) + + async for _ in PassThroughStreamingHandler.chunk_processor( + response=response, + request_body={"model": "claude-3-haiku"}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.GENERIC, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route="/bedrock/model/claude/invoke-with-response-stream", + route_streaming_logging=cost_callback, + ): + pass + + if deferred_dispatch_armed: + (parked_cost_callback,) = logging_obj._deferred_stream_complete_args + await parked_cost_callback + else: + await GLOBAL_LOGGING_WORKER.flush() + + assert reservation["callback_bound"] is True + assert claimed_when_the_callback_ran == [True] + + +@pytest.mark.asyncio +async def test_chunk_processor_leaves_the_budget_reservation_for_the_request_end_release_when_the_cost_callback_cannot_be_enqueued(): + reservation = {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + response = _make_streaming_response([b"event-1", b"event-2"]) + logging_obj = _unarmed_logging_obj() + logging_obj.litellm_params = {"metadata": {"user_api_key_budget_reservation": reservation}} + + def refuse_to_enqueue(async_coroutine): + async_coroutine.close() + raise RuntimeError("logging worker is shutting down") + + with patch.object(GLOBAL_LOGGING_WORKER, "ensure_initialized_and_enqueue", side_effect=refuse_to_enqueue): + async for _ in PassThroughStreamingHandler.chunk_processor( + response=response, + request_body={"model": "claude-3-haiku"}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.GENERIC, + start_time=datetime.now(), + passthrough_success_handler_obj=MagicMock(), + url_route="/bedrock/model/claude/invoke-with-response-stream", + route_streaming_logging=AsyncMock(), + ): + pass + + assert reservation["callback_bound"] is False diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py index 1d2d7d4d5c3..46f024366ec 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_ai_batch_passthrough.py @@ -462,30 +462,6 @@ class TestVertexAIBatchPassthroughHandler: assert mock_store.call_args[1]["unified_object_id"] assert mock_store.call_args[1]["is_batch_create"] is expected - def test_batch_cost_calculation_integration(self): - """Single Vertex AI response → non-zero cost with correct token counts.""" - from litellm.batches.batch_utils import calculate_vertex_ai_batch_cost_and_usage - - vertex_ai_batch_responses = [ - { - "response": { - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 5, - "totalTokenCount": 15, - } - } - } - ] - - result = calculate_vertex_ai_batch_cost_and_usage( - vertex_ai_batch_responses, model_name="gemini-2.0-flash-001" - ) - - assert result.usage.total_tokens == 15 - assert result.usage.prompt_tokens == 10 - assert result.usage.completion_tokens == 5 - assert result.cost > 0, "batch_cost_calculator should return a non-zero cost" def test_batch_response_transformation(self): """Test transformation of Vertex AI batch responses to OpenAI format""" @@ -639,76 +615,7 @@ class TestVertexAIBatchCostCalculation: batch_cost_calculator — no VertexGeminiConfig transformation involved. """ - def test_should_aggregate_cost_and_usage_across_responses(self): - """Two successful responses → costs and token counts are summed.""" - from litellm.batches.batch_utils import calculate_vertex_ai_batch_cost_and_usage - responses = [ - { - "response": { - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 5, - "totalTokenCount": 15, - } - } - }, - { - "response": { - "usageMetadata": { - "promptTokenCount": 8, - "candidatesTokenCount": 3, - "totalTokenCount": 11, - } - } - }, - ] - - result = calculate_vertex_ai_batch_cost_and_usage( - responses, model_name="gemini-2.0-flash-001" - ) - - assert result.usage.prompt_tokens == 18 - assert result.usage.completion_tokens == 8 - assert result.usage.total_tokens == 26 - assert result.cost > 0, "batch_cost_calculator should return a non-zero cost" - - def test_should_skip_responses_with_null_response_body(self): - """Failed lines (response: None) are skipped without error.""" - from litellm.batches.batch_utils import calculate_vertex_ai_batch_cost_and_usage - - responses = [ - { - "response": { - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 5, - "totalTokenCount": 15, - } - } - }, - {"status": "JOB_STATE_FAILED", "response": None}, - { - "response": { - "usageMetadata": { - "promptTokenCount": 8, - "candidatesTokenCount": 3, - "totalTokenCount": 11, - } - } - }, - ] - - result = calculate_vertex_ai_batch_cost_and_usage( - responses, model_name="gemini-2.0-flash-001" - ) - - assert result.usage.prompt_tokens == 18 - assert result.usage.completion_tokens == 8 - assert result.usage.total_tokens == 26 - assert result.cost > 0 - assert result.successful_requests == 2 - assert result.failed_requests == 1 def test_should_return_zeros_for_empty_response_list(self): """Empty input → zero cost and zero usage.""" @@ -739,143 +646,4 @@ class TestVertexAIBatchCostCalculation: assert result.usage.completion_tokens == 0 assert result.usage.total_tokens == 0 - @pytest.mark.asyncio - async def test_openai_shaped_output_records_nonzero_cost_and_usage(self): - """ - Regression test for the bug where Vertex batch cost/usage was always 0. - After PR #25627 (transform_file_content_response), the GCS predictions.jsonl - is rewritten into OpenAI batch shape before the cost-tracking path sees it. - With disable_vertex_batch_output_transformation=False (default), the cost - dispatch must fall through to the generic aggregation path rather than - calling calculate_vertex_ai_batch_cost_and_usage (which only reads raw - usageMetadata fields). - """ - import litellm - from litellm.batches.batch_utils import calculate_batch_cost_and_usage - - openai_shaped_responses = [ - { - "id": "batch_req_abc123", - "custom_id": "request-1", - "response": { - "status_code": 200, - "request_id": "chatcmpl-xyz", - "body": { - "id": "chatcmpl-xyz", - "object": "chat.completion", - "model": "gemini-2.0-flash-001", - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "Hello!"}, - "finish_reason": "stop", - } - ], - "usage": { - "prompt_tokens": 10, - "completion_tokens": 5, - "total_tokens": 15, - }, - }, - }, - "error": None, - }, - { - "id": "batch_req_def456", - "custom_id": "request-2", - "response": { - "status_code": 200, - "request_id": "chatcmpl-uvw", - "body": { - "id": "chatcmpl-uvw", - "object": "chat.completion", - "model": "gemini-2.0-flash-001", - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "World!"}, - "finish_reason": "stop", - } - ], - "usage": { - "prompt_tokens": 8, - "completion_tokens": 3, - "total_tokens": 11, - }, - }, - }, - "error": None, - }, - ] - - original_flag = getattr( - litellm, "disable_vertex_batch_output_transformation", False - ) - try: - litellm.disable_vertex_batch_output_transformation = False - - result = await calculate_batch_cost_and_usage( - file_content_dictionary=openai_shaped_responses, - custom_llm_provider="vertex_ai", - model_name="gemini-2.0-flash-001", - ) - finally: - litellm.disable_vertex_batch_output_transformation = original_flag - - assert ( - result.usage.prompt_tokens == 18 - ), f"expected 18 prompt tokens, got {result.usage.prompt_tokens}" - assert ( - result.usage.completion_tokens == 8 - ), f"expected 8 completion tokens, got {result.usage.completion_tokens}" - assert ( - result.usage.total_tokens == 26 - ), f"expected 26 total tokens, got {result.usage.total_tokens}" - assert ( - result.cost > 0 - ), f"expected non-zero cost for completed Vertex batch, got {result.cost}" - - @pytest.mark.asyncio - async def test_raw_vertex_output_still_works_when_transformation_disabled(self): - """ - When disable_vertex_batch_output_transformation=True the GCS file is returned - as raw Vertex predictions.jsonl; the specialized reader must be used. - """ - import litellm - from litellm.batches.batch_utils import calculate_batch_cost_and_usage - - raw_vertex_responses = [ - { - "request": {"contents": [{"role": "user", "parts": [{"text": "hi"}]}]}, - "status": "", - "response": { - "candidates": [{"content": {"parts": [{"text": "Hello!"}]}}], - "usageMetadata": { - "promptTokenCount": 10, - "candidatesTokenCount": 5, - "totalTokenCount": 15, - }, - }, - "processed_time": "2026-01-01T00:00:00Z", - }, - ] - - original_flag = getattr( - litellm, "disable_vertex_batch_output_transformation", False - ) - try: - litellm.disable_vertex_batch_output_transformation = True - - result = await calculate_batch_cost_and_usage( - file_content_dictionary=raw_vertex_responses, - custom_llm_provider="vertex_ai", - model_name="gemini-2.0-flash-001", - ) - finally: - litellm.disable_vertex_batch_output_transformation = original_flag - - assert result.usage.prompt_tokens == 10 - assert result.usage.completion_tokens == 5 - assert result.usage.total_tokens == 15 - assert result.cost > 0, "raw Vertex shape should also produce non-zero cost" diff --git a/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py b/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py index 53ea761daa7..089c2d57594 100644 --- a/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py +++ b/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py @@ -11,8 +11,10 @@ from __future__ import annotations import json from types import SimpleNamespace +from typing import Final from unittest.mock import MagicMock +import httpx import pytest from fastapi import HTTPException from fastapi.exceptions import RequestValidationError @@ -95,6 +97,71 @@ async def test_openai_exception_handler_invalid_empty_code_defaults_to_500(): } +def _call_id_exception(headers): + return ProxyException( + message="bad input", + type="invalid_request_error", + param="model", + code=400, + headers=headers, + ) + + +@pytest.mark.asyncio +async def test_openai_exception_handler_copies_the_call_id_into_the_error_when_opted_in(monkeypatch): + """With include_call_id_in_error_body on, error.litellm_call_id is byte-identical to the + x-litellm-call-id header, so a pasted str(e) names the request to look up.""" + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"include_call_id_in_error_body": True}) + exc = _call_id_exception({"x-litellm-call-id": "call-8302"}) + + response = await openai_exception_handler(request=_make_request(), exc=exc) + body = json.loads(response.body) + + assert response.headers["x-litellm-call-id"] == "call-8302" + assert body == { + "error": { + "message": "bad input", + "type": "invalid_request_error", + "param": "model", + "code": "400", + "litellm_call_id": "call-8302", + } + } + + +@pytest.mark.asyncio +async def test_openai_exception_handler_leaves_the_error_alone_when_opted_out(monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + exc = _call_id_exception({"x-litellm-call-id": "call-8302"}) + + response = await openai_exception_handler(request=_make_request(), exc=exc) + body = json.loads(response.body) + + assert response.headers["x-litellm-call-id"] == "call-8302" + assert body == { + "error": { + "message": "bad input", + "type": "invalid_request_error", + "param": "model", + "code": "400", + } + } + + +@pytest.mark.asyncio +async def test_openai_exception_handler_never_fabricates_a_call_id(monkeypatch): + """An error raised before a call id exists (auth failures, say) carries no header, + and the body must not invent one.""" + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"include_call_id_in_error_body": True}) + exc = _call_id_exception({}) + + response = await openai_exception_handler(request=_make_request(), exc=exc) + body = json.loads(response.body) + + assert "x-litellm-call-id" not in response.headers + assert "litellm_call_id" not in body["error"] + + # --------------------------------------------------------------------------- # _close_dangling_otel_server_span # --------------------------------------------------------------------------- @@ -299,6 +366,39 @@ async def test_otel_unhandled_exception_handler_returns_500_generic_payload(): } +_DB_OUTAGE_503_BODY: Final = { + "error": { + "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", + "type": "no_db_connection", + "param": "None", + "code": "503", + } +} + + +def _raised_from(outer: Exception, cause: Exception) -> Exception: + try: + raise outer from cause + except Exception as chained: + return chained + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "exc", + [ + httpx.ConnectError("All connection attempts failed"), + _raised_from(RuntimeError("user read failed"), httpx.ConnectError("All connection attempts failed")), + ], + ids=["raw_connect_error", "connect_error_as_cause"], +) +async def test_otel_unhandled_exception_handler_answers_a_db_outage_with_503_no_db_connection(exc): + response = await otel_unhandled_exception_handler(request=_make_request(path="/v2/team/list"), exc=exc) + + assert response.status_code == 503 + assert json.loads(response.body) == _DB_OUTAGE_503_BODY + + @pytest.mark.asyncio async def test_otel_unhandled_exception_handler_reraises_proxy_exception_error(): """ProxyException / HTTPException / RequestValidationError are re-raised diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index e9695685ca5..9954351fa2e 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -818,6 +818,35 @@ async def test_proxy_startup_event_prunes_dead_workers_live_gauges(tmp_path): assert counter.exists() +@pytest.mark.asyncio +@pytest.mark.parametrize(("disable_model_info_refresh", "job_scheduled"), [(True, False), (False, True)]) +async def test_proxy_startup_event_honors_disable_model_info_refresh( + disable_model_info_refresh: bool, job_scheduled: bool +) -> None: + """``general_settings.disable_model_info_refresh: true`` keeps the proxy from polling every + OpenAI-compatible deployment's ``/v1/models`` in the background, so a proxy fronting a replay + fixture (or a metered upstream) makes only the calls its clients asked for.""" + scheduler = AsyncIOScheduler() + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} | { + "LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY": "true" + } + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.object(ps, "scheduler", scheduler), + patch.dict(ps.general_settings, {"disable_model_info_refresh": disable_model_info_refresh}), + ): + try: + async with proxy_startup_event(app=None): + job = scheduler.get_job("refresh_model_info") + finally: + if scheduler.running: + scheduler.shutdown(wait=False) + + assert (job is not None) is job_scheduled, ( + f"disable_model_info_refresh={disable_model_info_refresh} but refresh_model_info job is {job}" + ) + + def test_otel_global_provider_published_after_callback_init(): """The OTel V2 global-provider publish must run after callback initialization in ``proxy_startup_event``. diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 462489f48b0..b47cce43dcc 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -32,6 +32,7 @@ from litellm.proxy.proxy_server import ( _scrub_guardrail_inner, resolve_complexity_router_plugins, resolve_routing_plugins, + validate_deployment_access_windows, validate_deployment_complexity_router_placement, validate_deployment_max_agentic_loops, validate_auto_router_capability_limits, @@ -2197,6 +2198,34 @@ async def test_ProxyConfig_load_config_wires_general_settings_url_validation(tmp litellm.provider_url_destination_allowed_hosts = original_provider_hosts +@pytest.mark.asyncio +async def test_ssrf_block_message_names_a_config_section_load_config_honors(tmp_path, monkeypatch): + """Regression for LIT-8349: the remediation in the SSRF block message must point at a section that works.""" + from litellm.litellm_core_utils.url_utils import SSRFError, validate_url + + monkeypatch.setattr(litellm, "user_url_allowed_hosts", []) + monkeypatch.setattr(litellm, "user_url_validation", True) + with pytest.raises(SSRFError) as blocked: + validate_url("http://10.96.3.245:10002/agent.json") + section_match = re.search(r"add the host to `user_url_allowed_hosts` in (\w+)\.", str(blocked.value)) + assert section_match is not None, str(blocked.value) + section: Final = section_match.group(1) + assert section == "litellm_settings", f"block message points admins at {section}, which the docs contradict" + + f = tmp_path / "c.yaml" + f.write_text(f"model_list: []\n{section}:\n user_url_allowed_hosts:\n - '10.96.3.245:10002'\n") + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + await ProxyConfig().load_config(router=None, config_file_path=str(f)) + + assert litellm.user_url_allowed_hosts == ["10.96.3.245:10002"], f"{section} did not apply the allowlist" + assert validate_url("http://10.96.3.245:10002/agent.json") == ( + "http://10.96.3.245:10002/agent.json", + "10.96.3.245:10002", + ) + + @pytest.mark.asyncio async def test_ProxyConfig_load_config_wires_config_reload_interval(tmp_path, monkeypatch): """general_settings.proxy_config_reload_interval_seconds must reach the proxy_server @@ -4714,3 +4743,59 @@ def test_websearch_interception_settings_can_be_named_in_supported_db_objects(mo monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["models"]}) assert proxy_server.should_load_db_object(object_type="websearch_interception_settings") is False + + +def test_validate_deployment_access_windows_rejects_malformed_time(): + model = { + "model_name": "gpt-4o-shared", + "litellm_params": {"model": "gpt-4o"}, + "model_info": { + "access_windows": [ + {"start": "25:00", "end": "06:00", "timezone": "America/New_York", "team_ids": ["t"]} + ] + }, + } + + with pytest.raises(ValueError, match="access_windows") as exc_info: + validate_deployment_access_windows(model) + + assert "gpt-4o-shared" in str(exc_info.value) + + +def test_validate_deployment_access_windows_rejects_unknown_timezone(): + model = { + "model_name": "gpt-4o-shared", + "litellm_params": {"model": "gpt-4o"}, + "model_info": { + "access_windows": [ + {"start": "22:00", "end": "06:00", "timezone": "Mars/Olympus", "team_ids": ["t"]} + ] + }, + } + + with pytest.raises(ValueError, match="Mars/Olympus"): + validate_deployment_access_windows(model) + + +def test_validate_deployment_access_windows_accepts_valid_and_absent(): + assert ( + validate_deployment_access_windows( + { + "model_name": "gpt-4o-shared", + "litellm_params": {"model": "gpt-4o"}, + "model_info": { + "access_windows": [ + {"start": "22:00", "end": "06:00", "timezone": "America/New_York", "team_ids": ["t"]} + ] + }, + } + ) + is None + ) + assert validate_deployment_access_windows({"model_name": "m", "litellm_params": {"model": "m"}}) is None + assert ( + validate_deployment_access_windows( + {"model_name": "m", "litellm_params": {"model": "m"}, "model_info": {"id": "x"}} + ) + is None + ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py index 0e0025f5194..9df8e6f4d67 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py @@ -1,9 +1,7 @@ from __future__ import annotations -import json import math from datetime import datetime, timedelta, timezone -from types import MappingProxyType from typing import Final import pytest @@ -24,14 +22,12 @@ from litellm.proxy.common_utils.user_api_key_cache import ( ) from litellm.proxy.spend_tracking.budget_reservation import ( _get_team_member_budget_counter, - count_request_input_tokens, estimate_request_max_cost, + release_unbound_budget_reservation, reserve_budget_for_request, ) from litellm.proxy.utils import ProxyLogging from litellm.router import Router -from litellm.rust_bridge import bindings, configuration -from litellm.rust_bridge import token_counter as rust_token_counter from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo TOKEN_COUNTING_ROUTES: Final = ( @@ -220,244 +216,6 @@ def test_deployment_pricing_update_invalidates_cached_estimate() -> None: assert math.isclose(after, before * 1000) -ANTHROPIC_TOKENIZER_MODEL: Final = "claude-sonnet-4-5-20250929" -CL100K_MODEL: Final = "gpt-4" -O200K_MODEL: Final = "gpt-4o" -RUST_COUNTED_BODY: Final = {"model": ANTHROPIC_TOKENIZER_MODEL, "max_tokens": 16, "messages": ANTHROPIC_MESSAGES} -RUST_INPUT_TOKENS: Final = 4_321 -RUST_INPUT_TOKENS_BY_TOKENIZER: Final = MappingProxyType( - {"anthropic": RUST_INPUT_TOKENS, "cl100k_base": 1_234, "o200k_base": 2_345} -) - - -class _FakeDeclined(Exception): - pass - - -class _FakeUpstream(Exception): - pass - - -class _FakeNative: - RustBridgeDeclined = _FakeDeclined - RustUpstreamError = _FakeUpstream - - -class _RecordingCounter: - """Stands in for one native counter; records `(tokenizer, body)` on the shared factory.""" - - def __init__(self, factory: _RecordingFactory, tokenizer: rust_token_counter.RustTokenizer) -> None: - self.factory = factory - self.tokenizer = tokenizer - - async def acount_request(self, body: bytes) -> object: - self.factory.calls.append((self.tokenizer, body)) - return {"model": "", "input_tokens": RUST_INPUT_TOKENS_BY_TOKENIZER[self.tokenizer]} - - -class _RecordingFactory: - """Stands in for the native `TokenCounter` class: called with tokenizer JSON, or `from_*_ranks`.""" - - def __init__(self) -> None: - self.calls: list[tuple[rust_token_counter.RustTokenizer, bytes]] = [] - - def __call__(self, tokenizer_json: str) -> _RecordingCounter: - return _RecordingCounter(self, "anthropic") - - def from_cl100k_ranks(self, rank_file: str) -> _RecordingCounter: - return _RecordingCounter(self, "cl100k_base") - - def from_o200k_ranks(self, rank_file: str) -> _RecordingCounter: - return _RecordingCounter(self, "o200k_base") - - -class _DecliningCounter: - async def acount_request(self, body: bytes) -> object: - raise _FakeDeclined("unsupported content block") - - -class _DecliningFactory: - def __call__(self, tokenizer_json: str) -> _DecliningCounter: - return _DecliningCounter() - - def from_cl100k_ranks(self, rank_file: str) -> _DecliningCounter: - return _DecliningCounter() - - def from_o200k_ranks(self, rank_file: str) -> _DecliningCounter: - return _DecliningCounter() - - -@pytest.fixture -def rust_counter(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) - rust_token_counter._counter.cache_clear() - configuration.reset_rust_configuration() - yield - rust_token_counter.TOKEN_COUNTER.reset() - rust_token_counter._counter.cache_clear() - configuration.reset_rust_configuration() - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("route", "request_body"), - ( - ("/v1/messages", RUST_COUNTED_BODY), - ("/v1/chat/completions", {"model": ANTHROPIC_TOKENIZER_MODEL, "messages": ANTHROPIC_MESSAGES}), - ("/v1/completions", {"model": ANTHROPIC_TOKENIZER_MODEL, "prompt": "hi"}), - ("/v1/responses", {"model": ANTHROPIC_TOKENIZER_MODEL, "input": "hi"}), - ("/v1/embeddings", {"model": ANTHROPIC_TOKENIZER_MODEL, "input": ["hi"]}), - ("/v1/rerank", {"model": ANTHROPIC_TOKENIZER_MODEL, "query": "hi", "documents": ["a"]}), - ), -) -async def test_rust_count_replaces_python_tokenizing_on_every_llm_route( - rust_counter: None, route: str, request_body: dict -) -> None: - factory: Final = _RecordingFactory() - litellm.rust(True) - rust_token_counter.TOKEN_COUNTER.override(factory) - raw_body: Final = json.dumps(request_body).encode() - - counts: Final = await count_request_input_tokens( - request_body=request_body, route=route, llm_router=None, raw_body=raw_body - ) - - assert dict(counts) == {ANTHROPIC_TOKENIZER_MODEL: RUST_INPUT_TOKENS} - assert factory.calls == [("anthropic", raw_body)] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("model", (CL100K_MODEL, "azure/gpt-35-turbo", "gemini/gemini-2.5-pro", "my-router-alias")) -async def test_tiktoken_cl100k_models_are_counted_by_rust(rust_counter: None, model: str) -> None: - factory: Final = _RecordingFactory() - litellm.rust(True) - rust_token_counter.TOKEN_COUNTER.override(factory) - body: Final = {"model": model, "messages": ANTHROPIC_MESSAGES} - raw_body: Final = json.dumps(body).encode() - - counts: Final = await count_request_input_tokens( - request_body=body, route="/v1/chat/completions", llm_router=None, raw_body=raw_body - ) - - assert dict(counts) == {model: RUST_INPUT_TOKENS_BY_TOKENIZER["cl100k_base"]} - assert factory.calls == [("cl100k_base", raw_body)] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("model", (O200K_MODEL, "gpt-5", "o3", "gpt-4.1", "chatgpt-4o-latest")) -async def test_tiktoken_o200k_models_are_counted_by_rust(rust_counter: None, model: str) -> None: - factory: Final = _RecordingFactory() - litellm.rust(True) - rust_token_counter.TOKEN_COUNTER.override(factory) - body: Final = {"model": model, "messages": ANTHROPIC_MESSAGES} - raw_body: Final = json.dumps(body).encode() - - counts: Final = await count_request_input_tokens( - request_body=body, route="/v1/chat/completions", llm_router=None, raw_body=raw_body - ) - - assert dict(counts) == {model: RUST_INPUT_TOKENS_BY_TOKENIZER["o200k_base"]} - assert factory.calls == [("o200k_base", raw_body)] - - -@pytest.mark.asyncio -async def test_multi_model_request_counts_once_per_tokenizer_and_python_for_the_rest(rust_counter: None) -> None: - factory: Final = _RecordingFactory() - litellm.rust(True) - rust_token_counter.TOKEN_COUNTER.override(factory) - models: Final = ( - CL100K_MODEL, - ANTHROPIC_TOKENIZER_MODEL, - "gemini/gemini-2.5-pro", - O200K_MODEL, - "gpt-5", - "replicate/meta/llama-2-70b-chat", - ) - body: Final = {"model": list(models), "messages": ANTHROPIC_MESSAGES} - raw_body: Final = json.dumps(body).encode() - python_counts: Final = await count_request_input_tokens( - request_body=body, route="/v1/chat/completions", llm_router=None - ) - - counts: Final = await count_request_input_tokens( - request_body=body, route="/v1/chat/completions", llm_router=None, raw_body=raw_body - ) - - assert factory.calls == [("cl100k_base", raw_body), ("anthropic", raw_body), ("o200k_base", raw_body)] - assert dict(counts) == { - CL100K_MODEL: RUST_INPUT_TOKENS_BY_TOKENIZER["cl100k_base"], - "gemini/gemini-2.5-pro": RUST_INPUT_TOKENS_BY_TOKENIZER["cl100k_base"], - ANTHROPIC_TOKENIZER_MODEL: RUST_INPUT_TOKENS, - O200K_MODEL: RUST_INPUT_TOKENS_BY_TOKENIZER["o200k_base"], - "gpt-5": RUST_INPUT_TOKENS_BY_TOKENIZER["o200k_base"], - "replicate/meta/llama-2-70b-chat": python_counts["replicate/meta/llama-2-70b-chat"], - } - assert counts["replicate/meta/llama-2-70b-chat"] not in RUST_INPUT_TOKENS_BY_TOKENIZER.values() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("model", (ANTHROPIC_TOKENIZER_MODEL, CL100K_MODEL, O200K_MODEL)) -async def test_rust_decline_falls_back_to_python_count(rust_counter: None, model: str) -> None: - litellm.rust(True) - rust_token_counter.TOKEN_COUNTER.override(_DecliningFactory()) - body: Final = {**RUST_COUNTED_BODY, "model": model} - python_counts: Final = await count_request_input_tokens(request_body=body, route="/v1/messages", llm_router=None) - - counts: Final = await count_request_input_tokens( - request_body=body, - route="/v1/messages", - llm_router=None, - raw_body=json.dumps(body).encode(), - ) - - assert dict(counts) == dict(python_counts) - assert counts[model] not in RUST_INPUT_TOKENS_BY_TOKENIZER.values() - - -@pytest.mark.asyncio -async def test_disabled_rust_never_sees_the_raw_body(rust_counter: None) -> None: - factory: Final = _RecordingFactory() - litellm.rust(False) - rust_token_counter.TOKEN_COUNTER.override(factory) - body: Final = {"model": [ANTHROPIC_TOKENIZER_MODEL, CL100K_MODEL, O200K_MODEL], "messages": ANTHROPIC_MESSAGES} - - counts: Final = await count_request_input_tokens( - request_body=body, - route="/v1/chat/completions", - llm_router=None, - raw_body=json.dumps(body).encode(), - ) - - assert factory.calls == [] - assert set(counts) == {ANTHROPIC_TOKENIZER_MODEL, CL100K_MODEL, O200K_MODEL} - assert not set(counts.values()) & set(RUST_INPUT_TOKENS_BY_TOKENIZER.values()) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("model", ("replicate/meta/llama-2-70b-chat", "meta-llama/Llama-3-8b", "text-davinci-003")) -async def test_models_without_a_rust_tokenizer_stay_in_python( - rust_counter: None, monkeypatch: pytest.MonkeyPatch, model: str -) -> None: - monkeypatch.setattr( - litellm, "open_ai_chat_completion_models", litellm.open_ai_chat_completion_models | {"text-davinci-003"} - ) - factory: Final = _RecordingFactory() - litellm.rust(True) - rust_token_counter.TOKEN_COUNTER.override(factory) - body: Final = {"model": model, "messages": ANTHROPIC_MESSAGES} - python_counts: Final = await count_request_input_tokens( - request_body=body, route="/v1/chat/completions", llm_router=None - ) - - counts: Final = await count_request_input_tokens( - request_body=body, route="/v1/chat/completions", llm_router=None, raw_body=json.dumps(body).encode() - ) - - assert factory.calls == [] - assert dict(counts) == dict(python_counts) - assert counts[model] not in RUST_INPUT_TOKENS_BY_TOKENIZER.values() - - @pytest.mark.asyncio @pytest.mark.parametrize( "expiry_offset, expected_max_budget", @@ -546,3 +304,41 @@ async def test_team_member_reservation_counter_adds_temp_increase_to_live_team_d assert counter is not None assert counter.max_budget == expected_max_budget assert counter.fallback_spend == 0.5 + + +@pytest.mark.asyncio +async def test_reservation_starts_unbound_to_any_callback(): + reservation: Final = await _reserve("/v1/responses") + + assert reservation is not None + assert reservation["callback_bound"] is False + + +@pytest.mark.asyncio +async def test_release_unbound_budget_reservation_frees_the_counter(spend_counter_cache: DualCache): + counter_key: Final = f"spend:key:{TINY_BUDGET_KEY_TOKEN}" + reservation: Final = await _reserve_for_tiny_budget_key( + "/v1/chat/completions", {"model": "gpt-4o", "messages": [{"role": "user", "content": "hello"}]} + ) + assert reservation is not None + assert spend_counter_cache.in_memory_cache.get_cache(key=counter_key) == pytest.approx(reservation["reserved_cost"]) + + await release_unbound_budget_reservation(reservation) + + assert spend_counter_cache.in_memory_cache.get_cache(key=counter_key) == pytest.approx(0.0) + assert reservation["finalized"] is True + + +@pytest.mark.asyncio +async def test_release_unbound_budget_reservation_leaves_a_bound_one_to_its_callback(spend_counter_cache: DualCache): + counter_key: Final = f"spend:key:{TINY_BUDGET_KEY_TOKEN}" + reservation: Final = await _reserve_for_tiny_budget_key( + "/v1/chat/completions", {"model": "gpt-4o", "messages": [{"role": "user", "content": "hello"}]} + ) + assert reservation is not None + reservation["callback_bound"] = True + + await release_unbound_budget_reservation(reservation) + + assert spend_counter_cache.in_memory_cache.get_cache(key=counter_key) == pytest.approx(reservation["reserved_cost"]) + assert reservation["finalized"] is False diff --git a/tests/test_litellm/proxy/spend_tracking/test_input_tokens.py b/tests/test_litellm/proxy/spend_tracking/test_input_tokens.py new file mode 100644 index 00000000000..a1da6c79cd7 --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_input_tokens.py @@ -0,0 +1,29 @@ +"""Tests for input-token counting shared across the reservation path's models.""" + +from __future__ import annotations + +from types import MappingProxyType +from typing import Final + +import pytest + +from litellm.proxy.spend_tracking.input_tokens import ( + TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, + count_input_tokens, + count_input_tokens_for_model, +) + +CL100K_MODEL: Final = "gpt-4" + + +@pytest.mark.asyncio +async def test_large_input_is_still_counted() -> None: + request_body: Final = { + "model": CL100K_MODEL, + "messages": [{"role": "user", "content": "x" * (TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS + 1)}], + } + + counts: Final = await count_input_tokens(request_body=request_body, raw_body=None, models=(CL100K_MODEL,)) + + assert counts[CL100K_MODEL] == count_input_tokens_for_model(request_body=request_body, model=CL100K_MODEL) + assert isinstance(counts, MappingProxyType) diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 004f07da431..de413a86521 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -634,25 +634,6 @@ def test_equal_modeled_usage_is_zero_under_equivalent_model_names() -> None: assert _savings("claude-opus-5", "anthropic/claude-opus-5", usage, usage) == 0.0 -def test_baseline_is_priced_under_its_own_provider(): - """Two providers can serve the same bare model name at different rates, so dropping - the provider prices the baseline against a vendor the operator never named. Here it - decides whether routing reads as a saving or a loss.""" - usage = Usage(prompt_tokens=100_000, completion_tokens=10_000, total_tokens=110_000) - azure = compute_autorouter_savings( - baseline_model="azure_ai/deepseek-r1", - selected_model="claude-haiku-4-5", - selected_provider="anthropic", - usage=usage, - ) - deepseek = compute_autorouter_savings( - baseline_model="deepseek/deepseek-r1", - selected_model="claude-haiku-4-5", - selected_provider="anthropic", - usage=usage, - ) - assert azure != pytest.approx(deepseek) - assert azure > 0 > deepseek def test_unresolvable_baseline_remains_unknown(): diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 9de6679472e..e41c027d962 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -139,6 +139,15 @@ def _reconstruct_ui_where_from_sql(sql_query, params): where["cache_hit"] = "hit" elif cond == "(cache_hit IS NULL OR LOWER(cache_hit) != 'true')": where["cache_hit"] = "miss" + elif "call_type" in cond: + if "call_type NOT IN" in cond: + where["span_type"] = "llm" + elif "call_mcp_tool" in cond: + where["span_type"] = "mcp" + elif "call_type = 'asend_message'" in cond: + where["span_type"] = "agent" + elif "acreate_batch" in cond: + where["span_type"] = "batch" elif sess: where["session_id"] = {"contains": str(params[int(sess.group(1)) - 1]).strip("%")} elif status: @@ -3418,6 +3427,95 @@ async def test_ui_view_spend_logs_with_cache_hit_filter(client, monkeypatch): app.dependency_overrides.pop(ps.user_api_key_auth, None) +@pytest.mark.asyncio +async def test_ui_view_spend_logs_with_span_type_filter(client, monkeypatch): + base = { + "api_key": "sk-test-key", + "user": "test_user_1", + "team_id": "team1", + "spend": 0.05, + "startTime": datetime.datetime.now(timezone.utc).isoformat(), + "model": "gpt-4", + "status": "success", + } + mock_spend_logs = [ + {**base, "id": "log1", "request_id": "req-llm", "call_type": "acompletion"}, + {**base, "id": "log2", "request_id": "req-agent", "call_type": "asend_message"}, + {**base, "id": "log3", "request_id": "req-mcp", "call_type": "call_mcp_tool"}, + {**base, "id": "log4", "request_id": "req-batch", "call_type": "aretrieve_batch"}, + ] + + call_types_by_span = { + "llm": lambda ct: ct not in {"call_mcp_tool", "list_mcp_tools", "asend_message"} + and ct not in {"acreate_batch", "create_batch", "aretrieve_batch", "retrieve_batch"}, + "agent": lambda ct: ct == "asend_message", + "mcp": lambda ct: ct in {"call_mcp_tool", "list_mcp_tools"}, + "batch": lambda ct: ct + in {"acreate_batch", "create_batch", "aretrieve_batch", "retrieve_batch"}, + } + + def filter_by_span_type(where): + span_type = where.get("span_type") + if span_type is None: + return mock_spend_logs + return [log for log in mock_spend_logs if call_types_by_span[span_type](log["call_type"])] + + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + make_ui_spend_logs_mock_prisma(mock_spend_logs, filter_by_span_type), + ) + + start_date, end_date = _default_date_range() + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN + ) + try: + for span_type, expected_ids in [ + ("batch", ["req-batch"]), + ("llm", ["req-llm"]), + ("mcp", ["req-mcp"]), + ("agent", ["req-agent"]), + ]: + response = client.get( + "/spend/logs/ui", + params={ + "span_type": span_type, + "start_date": start_date, + "end_date": end_date, + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200, response.text + data = response.json() + assert data["total"] == len(expected_ids) + assert [row["request_id"] for row in data["data"]] == expected_ids + + response = client.get( + "/spend/logs/ui", + params={ + "start_date": start_date, + "end_date": end_date, + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200 + assert response.json()["total"] == 4 + + response = client.get( + "/spend/logs/ui", + params={ + "span_type": "invalid", + "start_date": start_date, + "end_date": end_date, + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 400 + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + @pytest.mark.asyncio async def test_ui_view_spend_logs_with_model(client, monkeypatch): mock_spend_logs = [ @@ -3789,179 +3887,7 @@ class TestSpendLogsPayload: } return mock_response - @pytest.mark.asyncio - async def test_spend_logs_payload_success_log_with_api_base(self, monkeypatch): - from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler - # Clear any env overrides that would change the recorded api_base - monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) - monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) - - litellm.callbacks = [_ProxyDBLogger(message_logging=False)] - # litellm._turn_on_debug() - - client = AsyncHTTPHandler() - - with ( - patch.object( - litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter, - "_insert_spend_log_to_db", - ) as mock_client, - patch.object(litellm.proxy.proxy_server, "prisma_client"), - patch.object(client, "post", side_effect=self.mock_anthropic_response), - ): - response = await litellm.acompletion( - model="claude-4-sonnet-20250514", - messages=[{"role": "user", "content": "Hello, world!"}], - metadata={"user_api_key_end_user_id": "test_user_1"}, - client=client, - ) - - assert response.choices[0].message.content == "Hi! My name is Claude." - - await _wait_for_mock_call(mock_client) - - kwargs = mock_client.call_args.kwargs - payload: SpendLogsPayload = kwargs["payload"] - expected_payload = SpendLogsPayload( - **{ - "request_id": "chatcmpl-34df56d5-4807-45c1-bb99-61e52586b802", - "call_type": "acompletion", - "api_key": "", - "cache_hit": "None", - "startTime": datetime.datetime( - 2025, 3, 24, 22, 2, 42, 975883, tzinfo=datetime.timezone.utc - ), - "endTime": datetime.datetime( - 2025, 3, 24, 22, 2, 42, 989132, tzinfo=datetime.timezone.utc - ), - "completionStartTime": datetime.datetime( - 2025, 3, 24, 22, 2, 42, 989132, tzinfo=datetime.timezone.utc - ), - "model": "claude-4-sonnet-20250514", - "user": "", - "team_id": "", - "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', - "cache_key": "Cache OFF", - "spend": 0.01383, - "total_tokens": 2598, - "prompt_tokens": 2095, - "completion_tokens": 503, - "request_tags": "[]", - "end_user": "test_user_1", - "api_base": "https://api.anthropic.com/v1/messages", - "model_group": "", - "model_id": "", - "requester_ip_address": None, - "custom_llm_provider": "anthropic", - "messages": "{}", - "response": "{}", - "proxy_server_request": "{}", - "status": "success", - "mcp_namespaced_tool_name": None, - "agent_id": None, - } - ) - - differences = _compare_nested_dicts( - payload, expected_payload, ignore_keys=ignored_keys - ) - if differences: - pytest.fail(f"Dictionary mismatch: {differences}") - - @pytest.mark.asyncio - async def test_spend_logs_payload_success_log_with_router(self, monkeypatch): - from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler - - # Clear any env overrides that would change the recorded api_base - monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) - monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) - - litellm.callbacks = [_ProxyDBLogger(message_logging=False)] - # litellm._turn_on_debug() - - client = AsyncHTTPHandler() - - router = Router( - model_list=[ - { - "model_name": "my-anthropic-model-group", - "litellm_params": { - "model": "claude-4-sonnet-20250514", - }, - "model_info": { - "id": "my-unique-model-id", - }, - } - ] - ) - - with ( - patch.object( - litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter, - "_insert_spend_log_to_db", - ) as mock_client, - patch.object(litellm.proxy.proxy_server, "prisma_client"), - patch.object(client, "post", side_effect=self.mock_anthropic_response), - ): - response = await router.acompletion( - model="my-anthropic-model-group", - messages=[{"role": "user", "content": "Hello, world!"}], - metadata={"user_api_key_end_user_id": "test_user_1"}, - client=client, - ) - - assert response.choices[0].message.content == "Hi! My name is Claude." - - await _wait_for_mock_call(mock_client) - - kwargs = mock_client.call_args.kwargs - payload: SpendLogsPayload = kwargs["payload"] - expected_payload = SpendLogsPayload( - **{ - "request_id": "chatcmpl-34df56d5-4807-45c1-bb99-61e52586b802", - "call_type": "acompletion", - "api_key": "", - "cache_hit": "None", - "startTime": datetime.datetime( - 2025, 3, 24, 22, 2, 42, 975883, tzinfo=datetime.timezone.utc - ), - "endTime": datetime.datetime( - 2025, 3, 24, 22, 2, 42, 989132, tzinfo=datetime.timezone.utc - ), - "completionStartTime": datetime.datetime( - 2025, 3, 24, 22, 2, 42, 989132, tzinfo=datetime.timezone.utc - ), - "model": "claude-4-sonnet-20250514", - "user": "", - "team_id": "", - "metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', - "cache_key": "Cache OFF", - "spend": 0.01383, - "total_tokens": 2598, - "prompt_tokens": 2095, - "completion_tokens": 503, - "request_tags": "[]", - "end_user": "test_user_1", - "api_base": "https://api.anthropic.com/v1/messages", - "model_group": "my-anthropic-model-group", - "model_id": "my-unique-model-id", - "requester_ip_address": None, - "custom_llm_provider": "anthropic", - "messages": "{}", - "response": "{}", - "proxy_server_request": "{}", - "status": "success", - "mcp_namespaced_tool_name": None, - "agent_id": None, - } - ) - - differences = _compare_nested_dicts( - payload, expected_payload, ignore_keys=ignored_keys - ) - if differences: - pytest.fail(f"Dictionary mismatch: {differences}") def _compare_nested_dicts( diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py index 86bf896188f..b3913079bb2 100644 --- a/tests/test_litellm/proxy/test_budget_reservation.py +++ b/tests/test_litellm/proxy/test_budget_reservation.py @@ -39,8 +39,6 @@ from litellm.proxy.common_utils.user_api_key_cache import ( model_access_group_spend_counter_key, ) from litellm.proxy.spend_tracking.budget_reservation import ( - TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, - _approximate_input_size, _get_model_access_group_budget_counters, estimate_request_max_cost, get_budget_window_start, @@ -49,6 +47,10 @@ from litellm.proxy.spend_tracking.budget_reservation import ( release_budget_reservation_on_cancel, reserve_budget_for_request, ) +from litellm.proxy.spend_tracking.input_tokens import ( + TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, + _approximate_input_size, +) from litellm.proxy.utils import ProxyLogging from litellm.router import Router from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget diff --git a/tests/test_litellm/proxy/test_bug_report_config.py b/tests/test_litellm/proxy/test_bug_report_config.py new file mode 100644 index 00000000000..6cffa55781e --- /dev/null +++ b/tests/test_litellm/proxy/test_bug_report_config.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +from collections.abc import Iterator, Mapping + +import pytest + +from litellm.proxy import proxy_server +from litellm.proxy.bug_report_config import build_proxy_bug_report, build_proxy_environment_report, safe_config_lines + +CUSTOMER_STRINGS = ( + "acme", + "sk-live-secret", + "hunter2", + "postgres://", + "10.0.0.7", +) + +CUSTOMER_CONFIG: Mapping[str, object] = { + "model_list": [ + { + "model_name": "acme-prod-gpt4", + "litellm_params": { + "model": "azure/acme-gpt4o-deployment", + "api_base": "https://acme-eastus.openai.azure.com", + "api_key": "sk-live-secret-1", + "rpm": 600, + "acme_extra_param": "acme", + }, + }, + { + "model_name": "acme-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-live-secret-2"}, + }, + { + "model_name": "acme-backup", + "litellm_params": {"model": "azure/acme-backup-deployment", "api_key": "sk-live-secret-3"}, + }, + {"model_name": "acme-bare", "litellm_params": {"model": "acme-custom-model"}}, + ], + "litellm_settings": { + "callbacks": ["langfuse", "acme_hooks.audit_logger"], + "drop_params": True, + "num_retries": 3, + "acme_internal_flag": True, + "cache": True, + "cache_params": { + "type": "redis", + "host": "10.0.0.7", + "port": 6379, + "password": "hunter2", + "acme_cache_option": "acme", + }, + }, + "router_settings": { + "routing_strategy": "latency-based-routing", + "redis_host": "10.0.0.7", + "acme_router_option": True, + }, + "guardrails": [ + {"guardrail_name": "acme-pii-mask", "litellm_params": {"guardrail": "presidio", "mode": "pre_call"}}, + { + "guardrail_name": "acme-policy", + "litellm_params": {"guardrail": "acme_guardrails.PolicyCheck", "api_key": "sk-live-secret-4"}, + }, + ], + "environment_variables": {"ACME_PROD_OPENAI_KEY": "sk-live-secret-5", "ACME_TENANT": "acme"}, +} + +CUSTOMER_GENERAL_SETTINGS: Mapping[str, object] = { + "master_key": "sk-live-secret-master", + "database_url": "postgres://user:hunter2@10.0.0.7/litellm", + "key_management_system": "aws_secret_manager", + "store_model_in_db": True, + "health_check_interval": 300, + "acme_sso_tenant": "acme-prod", +} + + +def test_safe_config_lines_keep_only_flags_and_litellm_defined_values(): + lines = safe_config_lines(CUSTOMER_CONFIG, CUSTOMER_GENERAL_SETTINGS) + + assert lines == ( + "general_settings.key_management_system = aws_secret_manager", + "general_settings.store_model_in_db = true", + "litellm_settings.callbacks = [langfuse]", + "litellm_settings.drop_params = true", + "litellm_settings.cache = true", + "litellm_settings.cache_params.type = redis", + "router_settings.routing_strategy = latency-based-routing", + "guardrails[0].litellm_params.guardrail = presidio", + "guardrails[0].litellm_params.mode = pre_call", + "model_list[*].provider = [azure, openai]", + ) + assert not any(customer_string in "\n".join(lines) for customer_string in CUSTOMER_STRINGS) + + +@pytest.mark.parametrize( + ("config", "expected_lines"), + [ + ({"router_settings": {"routing_strategy": "acme-strategy"}}, ()), + ({"litellm_settings": {"cache_params": {"type": "acme-cache"}}}, ()), + ( + {"litellm_settings": {"success_callback": ["acme_logger", "langsmith"]}}, + ("litellm_settings.success_callback = [langsmith]",), + ), + ({"litellm_settings": {"callbacks": ["acme_hooks.audit_logger"]}}, ()), + ], +) +def test_string_values_show_only_when_litellm_defines_them( + config: Mapping[str, object], expected_lines: tuple[str, ...] +): + assert safe_config_lines(config, {}) == expected_lines + + +def test_secrets_numbers_and_unknown_values_leave_no_line(): + general_settings: Mapping[str, object] = { + "master_key": "sk-live-secret-master", + "health_check_interval": 300, + "store_model_in_db": object(), + "alerting": {"acme": "webhook"}, + "background_health_checks": False, + } + + assert safe_config_lines({}, general_settings) == ("general_settings.background_health_checks = false",) + + +def test_credential_keys_never_render_even_when_the_secret_equals_a_litellm_token(): + config: Mapping[str, object] = { + "litellm_settings": { + "openai_key": "openai", + "token": "langfuse", + "api_base": "azure", + "callbacks": ["langfuse"], + "cache_params": {"type": "redis", "password": "redis", "host": "openai", "qdrant_api_key": "qdrant"}, + }, + "router_settings": { + "routing_strategy": "simple-shuffle", + "redis_password": "simple-shuffle", + "redis_url": "redis", + }, + "guardrails": [ + { + "litellm_params": { + "guardrail": "presidio", + "api_key": "presidio", + "auth_token": "pre_call", + "client_secret": "openai", + "credentials": ["openai", "azure"], + } + } + ], + } + general_settings: Mapping[str, object] = { + "master_key": "redis", + "database_url": "openai", + "alert_to_webhook_url": "langfuse", + "key_management_system": "aws_secret_manager", + "use_azure_key_vault": True, + } + + assert safe_config_lines(config, general_settings) == ( + "general_settings.key_management_system = aws_secret_manager", + "general_settings.use_azure_key_vault = true", + "litellm_settings.callbacks = [langfuse]", + "litellm_settings.cache_params.type = redis", + "router_settings.routing_strategy = simple-shuffle", + "guardrails[0].litellm_params.guardrail = presidio", + ) + + +def test_malformed_sections_produce_no_lines(): + config: Mapping[str, object] = { + "litellm_settings": "acme", + "router_settings": ["acme"], + "guardrails": {"acme": {"litellm_params": {"guardrail": "presidio"}}}, + "model_list": "acme", + "environment_variables": None, + } + + assert safe_config_lines(config, {}) == () + + +@pytest.fixture +def loaded_proxy_config(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + previous_config = proxy_server.proxy_config.get_config_state() + proxy_server.proxy_config.update_config_state(config=CUSTOMER_CONFIG) + monkeypatch.setattr(proxy_server, "general_settings", dict(CUSTOMER_GENERAL_SETTINGS)) + yield + proxy_server.proxy_config.update_config_state(config=previous_config) + + +@pytest.mark.usefixtures("loaded_proxy_config") +def test_build_proxy_bug_report_reads_the_loaded_proxy_config(): + report = build_proxy_bug_report(RuntimeError("boom"), stream=False) + + assert report.environment.surface == "proxy" + assert report.stream is False + assert report.environment.config_lines == safe_config_lines(CUSTOMER_CONFIG, CUSTOMER_GENERAL_SETTINGS) + + +@pytest.mark.usefixtures("loaded_proxy_config") +def test_proxy_environment_report_matches_the_bug_report_environment(): + environment = build_proxy_environment_report() + + assert environment == build_proxy_bug_report(RuntimeError("boom")).environment + assert environment.config_lines == safe_config_lines(CUSTOMER_CONFIG, CUSTOMER_GENERAL_SETTINGS) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 0b872400be0..5b9cd761dda 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -4,6 +4,7 @@ import datetime import json from types import MappingProxyType, SimpleNamespace from typing import AsyncGenerator, Callable, Final, Iterator, Optional, Sequence +from urllib.parse import unquote_plus from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -13,6 +14,12 @@ from fastapi.responses import JSONResponse, StreamingResponse import litellm from litellm._uuid import uuid +from litellm.litellm_core_utils.bug_report import ( + DISABLE_ENV_VAR, + ISSUE_URL_BASE, + bug_report_notice, + build_bug_report, +) from litellm.constants import ( CLIENT_REQUESTED_MODEL_SCOPE_KEY, MAX_LITELLM_CALL_ID_LENGTH, @@ -28,10 +35,12 @@ from litellm.proxy.common_request_processing import ( _buffer_first_chunk_honoring_disconnect, _cancel_llm_call_on_client_disconnect, _ClientDisconnectedBeforeFirstChunk, + attach_guardrail_information, _extract_error_from_sse_chunk, _get_cost_breakdown_from_logging_obj, CostBreakdownHeaderValues, _has_attribute_error_in_chain, + include_guardrail_response_requested, _is_azure_model_router_request, open_sse_before_first_byte, resolve_litellm_call_id, @@ -54,6 +63,132 @@ from litellm.proxy.utils import ProxyLogging from litellm.router import Router +def test_attach_guardrail_information_copies_recorded_entries_onto_model_response(): + recorded = [ + {"guardrail_name": "first", "guardrail_status": "success"}, + {"guardrail_name": "second", "guardrail_status": "success"}, + ] + response = litellm.ModelResponse() + + result = attach_guardrail_information( + response=response, + request_data={"metadata": {"standard_logging_guardrail_information": recorded}}, + ) + + assert isinstance(result, litellm.ModelResponse) + assert result.model_dump()["guardrail_information"] == recorded + assert "guardrail_information" not in response.model_dump() + + +def test_attach_guardrail_information_reports_empty_list_when_nothing_ran(): + response = litellm.ModelResponse() + + result = attach_guardrail_information(response=response, request_data={}) + + assert isinstance(result, litellm.ModelResponse) + assert result.model_dump()["guardrail_information"] == [] + assert "guardrail_information" not in response.model_dump() + + +def test_attach_guardrail_information_sets_key_on_dict_response(): + recorded = [{"guardrail_name": "first", "guardrail_status": "success"}] + response = {"id": "x"} + + result = attach_guardrail_information( + response=response, + request_data={"metadata": {"standard_logging_guardrail_information": recorded}}, + ) + + assert isinstance(result, dict) + assert result == {"id": "x", "guardrail_information": recorded} + assert response == {"id": "x"} + + +def test_attach_guardrail_information_redacts_matched_content(): + recorded = [ + { + "guardrail_name": "cf", + "guardrail_status": "success", + "guardrail_response": [ + {"type": "blocked_word", "keyword": "secret-word", "action": "MASK"} + ], + "match_details": [{"snippet": "secret-word", "detection_method": "keyword"}], + } + ] + + result = attach_guardrail_information( + response={"id": "x"}, + request_data={"metadata": {"standard_logging_guardrail_information": recorded}}, + ) + + assert isinstance(result, dict) + guardrail_information = result["guardrail_information"] + assert isinstance(guardrail_information, list) + assert guardrail_information[0]["guardrail_response"][0]["keyword"] == "[REDACTED]" + assert guardrail_information[0]["match_details"][0]["snippet"] == "[REDACTED]" + assert guardrail_information[0]["match_details"][0]["detection_method"] == "keyword" + assert "secret-word" not in json.dumps(result) + + +def test_attach_guardrail_information_leaves_cached_dict_response_untouched(): + recorded = [{"guardrail_name": "cf", "guardrail_status": "success"}] + cached = {"id": "x", "content": []} + + result = attach_guardrail_information( + response=cached, + request_data={ + "metadata": { + "include_guardrail_response": True, + "standard_logging_guardrail_information": recorded, + } + }, + ) + + assert "guardrail_information" not in cached + assert result is not cached + assert isinstance(result, dict) + assert result["guardrail_information"] == recorded + + original = litellm.ModelResponse() + copied = attach_guardrail_information( + response=original, + request_data={"metadata": {"standard_logging_guardrail_information": recorded}}, + ) + + assert "guardrail_information" not in original.model_dump() + assert isinstance(copied, litellm.ModelResponse) + assert copied.model_dump()["guardrail_information"] == recorded + + +def test_include_guardrail_response_requested_reads_flag_from_metadata_when_router_seeded_litellm_metadata(): + recorded = [ + {"guardrail_name": "first", "guardrail_status": "success"}, + {"guardrail_name": "second", "guardrail_status": "success"}, + ] + request_data = { + "metadata": { + "include_guardrail_response": True, + "standard_logging_guardrail_information": recorded, + }, + "litellm_metadata": {}, + } + + assert include_guardrail_response_requested(request_data) is True + + response = litellm.ModelResponse() + result = attach_guardrail_information(response=response, request_data=request_data) + + assert isinstance(result, litellm.ModelResponse) + assert result.model_dump()["guardrail_information"] == recorded + + +def test_include_guardrail_response_requested_is_false_without_exact_true(): + assert include_guardrail_response_requested( + {"metadata": {"include_guardrail_response": "true"}, "litellm_metadata": {}} + ) is False + assert include_guardrail_response_requested({}) is False + + class TestProxyBaseLLMRequestProcessing: @pytest.mark.asyncio async def test_base_passthrough_process_llm_request_preserves_litellm_headers_for_non_streaming_response( @@ -2343,6 +2478,39 @@ class TestCommonRequestProcessingHelpers: assert isinstance(response, JSONResponse) assert response.headers["x-litellm-model-id"] == "fallback-deployment" + @staticmethod + async def _first_chunk_error_response(**create_response_kwargs): + async def mock_generator(): + yield 'data: {"error": {"code": 403, "message": "forbidden"}}\n\n' + yield "data: [DONE]\n\n" + + return await create_response( + mock_generator(), + "text/event-stream", + {"x-litellm-call-id": "call-8302"}, + **create_response_kwargs, + ) + + async def test_create_response_first_chunk_error_carries_the_call_id_when_opted_in(self): + """A stream that fails on its first chunk answers as JSON, and with + include_call_id_in_error_body on that JSON names the request like the + non-streaming error path does, byte-identical to the header.""" + response = await self._first_chunk_error_response(general_settings={"include_call_id_in_error_body": True}) + + assert isinstance(response, JSONResponse) + assert response.status_code == 403 + assert response.headers["x-litellm-call-id"] == "call-8302" + assert json.loads(response.body) == { + "error": {"code": 403, "message": "forbidden", "litellm_call_id": "call-8302"} + } + + async def test_create_response_first_chunk_error_body_is_unchanged_by_default(self): + response = await self._first_chunk_error_response() + + assert isinstance(response, JSONResponse) + assert response.headers["x-litellm-call-id"] == "call-8302" + assert json.loads(response.body) == {"error": {"code": 403, "message": "forbidden"}} + async def test_create_streaming_response_disables_proxy_buffering(self): """Regression for #28384: every StreamingResponse create_response returns must carry the headers that stop nginx/ingress/Envoy from buffering the @@ -4257,6 +4425,21 @@ class TestHandleLLMApiExceptionRetryAfter: proxy_exc = await self._invoke(ValueError("some other failure")) assert "retry-after" not in proxy_exc.headers + async def test_handle_llm_api_exception_strips_bug_report_notice_from_client_message(self, caplog): + report = build_bug_report(RuntimeError("boom"), surface="sdk") + notice = bug_report_notice(report) + exc = litellm.APIConnectionError( + message=f"boom\n{notice}", + model="gpt-4o", + llm_provider="openai", + ) + + with caplog.at_level("ERROR"): + proxy_exc = await self._invoke(exc) + + assert ISSUE_URL_BASE not in proxy_exc.message + assert ISSUE_URL_BASE in caplog.text + async def test_handle_llm_api_exception_retry_after_survives_callback_headers(self): from litellm.types.router import RouterRateLimitError @@ -8979,6 +9162,60 @@ class TestDetachedStreamFailureHook: assert [call["original_exception"] for call in recorder.calls] == [failure] +class TestPostCallMaskedOutputReachesDeferredLogging: + @pytest.mark.asyncio + async def test_non_streaming_records_the_masked_response_before_deferred_logging_fires(self, monkeypatch): + from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY + from litellm.types.utils import Choices, Message, ModelResponse + + logging_obj = MagicMock() + logging_obj.litellm_call_id = "lit-8325-call" + logging_obj._defer_async_logging = False + logging_obj._on_deferred_stream_complete = None + logging_obj.cost_breakdown = None + logging_obj.model_call_details = {} + recorded_at_enqueue: dict[str, object] = {} + logging_obj._enqueue_deferred_logging = lambda: recorded_at_enqueue.update(logging_obj.model_call_details) + + processor = ProxyBaseLLMRequestProcessing(data={"model": "oa", "litellm_logging_obj": logging_obj}) + + def mask(data, user_api_key_dict, response): + response.choices[0].message.content = "Card: " + return response + + proxy_logging_obj = MagicMock(spec=ProxyLogging) + proxy_logging_obj.during_call_hook = AsyncMock(return_value=None) + proxy_logging_obj.update_request_status = AsyncMock(return_value=None) + proxy_logging_obj.post_call_success_hook = AsyncMock(side_effect=mask) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value=None) + + async def fake_route_request(**kwargs): + async def call(): + return ModelResponse( + choices=[Choices(index=0, message=Message(content="Card: 4111 1111 1111 1111", role="assistant"))] + ) + + return call() + + monkeypatch.setattr(litellm.proxy.common_request_processing, "route_request", fake_route_request) + + result = await processor.base_process_llm_request( + request=Request(scope={"type": "http", "headers": []}), + fastapi_response=Response(), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + route_type="acompletion", + proxy_logging_obj=proxy_logging_obj, + general_settings={}, + proxy_config=MagicMock(spec=ProxyConfig), + select_data_generator=MagicMock(), + is_streaming_request=False, + skip_pre_call_logic=True, + ) + + assert result.choices[0].message.content == "Card: " + assert recorded_at_enqueue[SERVED_OUTPUT_TEXTS_KEY] == ("Card: ",) + + class TestStreamingResponseHeadersFollowFallback: """LIT-6767: the streaming branch has to publish the deployment that served the stream.""" @@ -9122,6 +9359,62 @@ class TestStreamingResponseHeadersFollowFallback: assert result.status_code == 400 assert result.headers["x-litellm-applied-guardrails"] == "stream-blocker" + @pytest.mark.asyncio + async def test_streaming_first_chunk_error_carries_the_call_id_when_opted_in(self, monkeypatch): + """The opt-in reaches the streaming path through base_process_llm_request, so a stream + that fails on its first chunk answers with the call id inside its JSON error body, + byte-identical to the x-litellm-call-id header.""" + + def select_data_generator(**kwargs): + async def generator(): + yield 'data: {"error": {"code": 403, "message": "forbidden"}}\n\n' + yield "data: [DONE]\n\n" + + return generator() + + logging_obj = MagicMock() + logging_obj.litellm_call_id = "lit-8302-call" + logging_obj._defer_async_logging = False + logging_obj._on_deferred_stream_complete = None + logging_obj.cost_breakdown = None + processor = ProxyBaseLLMRequestProcessing( + data={"model": "oa", "stream": True, "litellm_logging_obj": logging_obj} + ) + + proxy_logging_obj = MagicMock(spec=ProxyLogging) + proxy_logging_obj.during_call_hook = AsyncMock(return_value=None) + proxy_logging_obj.update_request_status = AsyncMock(return_value=None) + proxy_logging_obj.post_call_success_hook = AsyncMock( + side_effect=lambda data, user_api_key_dict, response: response + ) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + async def fake_route_request(**kwargs): + async def call(): + return SimpleNamespace(_hidden_params={}, fallback_headers_adopted=False) + + return call() + + monkeypatch.setattr(litellm.proxy.common_request_processing, "route_request", fake_route_request) + + result = await processor.base_process_llm_request( + request=Request(scope={"type": "http", "headers": []}), + fastapi_response=Response(), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + route_type="acompletion", + proxy_logging_obj=proxy_logging_obj, + general_settings={"include_call_id_in_error_body": True}, + proxy_config=MagicMock(spec=ProxyConfig), + select_data_generator=select_data_generator, + is_streaming_request=True, + skip_pre_call_logic=True, + ) + + assert isinstance(result, JSONResponse) + assert result.status_code == 403 + assert result.headers["x-litellm-call-id"] == "lit-8302-call" + assert json.loads(result.body)["error"]["litellm_call_id"] == "lit-8302-call" + class _MessagesFallbackStream: def __init__(self) -> None: @@ -9285,6 +9578,94 @@ async def test_handle_llm_api_exception_forwards_litellm_response_headers_when_r assert exc_info.value.headers["llm_provider-x-request-id"] == "req_openai_400" +@pytest.mark.asyncio +async def test_handle_llm_api_exception_logs_bug_report_for_unmapped_error( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +): + monkeypatch.delenv(DISABLE_ENV_VAR, raising=False) + processor = ProxyBaseLLMRequestProcessing( + data={ + "proxy_server_request": {"url": "https://example.test/v1/chat/completions?debug=true"}, + "model": "acme-prod-gpt4", + "custom_llm_provider": "openai", + "stream": True, + } + ) + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + with caplog.at_level("ERROR", logger="LiteLLM Proxy"): + with pytest.raises(ProxyException): + await processor._handle_llm_api_exception( + e=RuntimeError("unmapped for user@example.com"), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + ) + + issue_url = next(word for word in caplog.text.split() if word.startswith(ISSUE_URL_BASE)) + assert "Endpoint / call: /v1/chat/completions" in unquote_plus(issue_url) + assert "Provider: openai" in unquote_plus(issue_url) + assert "Stream: true" in unquote_plus(issue_url) + assert "acme-prod-gpt4" not in unquote_plus(issue_url) + assert "user@example.com" not in unquote_plus(issue_url) + + +@pytest.mark.asyncio +async def test_handle_llm_api_exception_bug_report_drops_unknown_route( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +): + monkeypatch.delenv(DISABLE_ENV_VAR, raising=False) + processor = ProxyBaseLLMRequestProcessing( + data={"proxy_server_request": {"url": "https://example.test/v1/files/file-customer-123/content"}} + ) + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + with caplog.at_level("ERROR", logger="LiteLLM Proxy"): + with pytest.raises(ProxyException): + await processor._handle_llm_api_exception( + e=RuntimeError("unmapped"), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + ) + + issue_url = next(word for word in caplog.text.split() if word.startswith(ISSUE_URL_BASE)) + assert "Endpoint / call: unknown" in unquote_plus(issue_url) + assert "file-customer-123" not in unquote_plus(issue_url) + + +@pytest.mark.asyncio +async def test_handle_llm_api_exception_skips_bug_report_for_provider_status( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +): + monkeypatch.delenv(DISABLE_ENV_VAR, raising=False) + + class ProviderRateLimitError(Exception): + def __init__(self, message: str): + super().__init__(message) + self.status_code = 429 + + processor = ProxyBaseLLMRequestProcessing(data={}) + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + with caplog.at_level("ERROR", logger="LiteLLM Proxy"): + with pytest.raises(ProxyException): + await processor._handle_llm_api_exception( + e=ProviderRateLimitError("rate limited"), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + ) + + assert ISSUE_URL_BASE not in caplog.text + + class TestBackgroundResponseRetrievalGovernance: """LIT-7175: retrieving a background Response attaches the model's post_call policy pipelines.""" diff --git a/tests/test_litellm/proxy/test_component_allowlists.py b/tests/test_litellm/proxy/test_component_allowlists.py index 3073908fa54..3641a2d9be9 100644 --- a/tests/test_litellm/proxy/test_component_allowlists.py +++ b/tests/test_litellm/proxy/test_component_allowlists.py @@ -23,8 +23,10 @@ import (which raises on a non-postgres ``DATABASE_URL`` scheme and can mint an RDS IAM token when ``IAM_TOKEN_DB_AUTH`` is set). """ +import json import os import sys +from typing import Final # Importing ``litellm.proxy.proxy_server`` runs its module-level setup, which # reads ``DATABASE_URL`` (Prisma) and ``LITELLM_MASTER_KEY``. Tier-zero CI @@ -49,17 +51,10 @@ _REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", if _REPO_ROOT not in sys.path: sys.path.insert(0, _REPO_ROOT) -from backend.routes.allowlist import ( - BACKEND_EXACT_PATHS, - BACKEND_MOUNT_PATHS, - BACKEND_PATH_PREFIXES, -) -from gateway.routes.allowlist import ( - GATEWAY_EXACT_PATHS, - GATEWAY_MOUNT_PATHS, - GATEWAY_PATH_PREFIXES, -) +from backend.routes.allowlist import BACKEND_MOUNT_PATHS +from gateway.routes.allowlist import GATEWAY_MOUNT_PATHS from litellm.proxy.proxy_server import app +from tests.test_litellm_rust.support.child_interpreter import run_child_interpreter for _key, _previous in _PRE_EXISTING_ENV.items(): if _previous is None: @@ -87,40 +82,69 @@ for _key, _previous in _PRE_DB_ENV.items(): os.environ[_key] = _previous -def _component_paths(routes, exact_paths, path_prefixes) -> set[str]: - """Reproduce ``gateway.main._is_gateway_route`` / ``backend.main._is_backend_route``.""" - out: set[str] = set() - for route in routes: - if isinstance(route, Mount): - continue - path = getattr(route, "path", None) - if path is None: - continue - if path in exact_paths or any(path.startswith(p) for p in path_prefixes): - out.add(path) - return out +_COVERAGE_PROBE: Final = """ +import json, os, sys +sys.path.insert(0, os.environ["LITELLM_COMPONENT_ALLOWLIST_REPO_ROOT"]) +from fastapi.routing import Mount +from backend.routes.allowlist import BACKEND_EXACT_PATHS, BACKEND_PATH_PREFIXES +from gateway.routes.allowlist import GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES +from litellm.proxy._lazy_features import loaded_lazy_modules +from litellm.proxy.proxy_server import app + +all_paths = { + r.path for r in app.router.routes + if not isinstance(r, Mount) and getattr(r, "path", None) is not None +} + + +def covered(exact, prefixes): + return {p for p in all_paths if p in exact or any(p.startswith(x) for x in prefixes)} + + +json.dump({ + "lazy_loaded": sorted(loaded_lazy_modules(app)), + "route_count": len(all_paths), + "uncovered": sorted(all_paths - ( + covered(GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES) + | covered(BACKEND_EXACT_PATHS, BACKEND_PATH_PREFIXES) + )), +}, sys.stdout) +""" def test_gateway_plus_backend_covers_full_app(): - """Every route on the proxy app must be served by gateway or backend.""" - all_paths = { - getattr(r, "path") - for r in app.router.routes - if not isinstance(r, Mount) and getattr(r, "path", None) is not None - } - gateway_paths = _component_paths( - app.router.routes, GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES + """Every route on the proxy app must be served by gateway or backend. + + ``gateway.main`` and ``backend.main`` trim the route table once, inside the + lifespan, so the set this has to cover is the one registered at startup. A + lazy feature appends its router on demand, after that trim, and whether a + sibling test in the same xdist worker has triggered one is not something + this test can control. Measuring in a fresh interpreter is what makes the + route table deterministic; nothing is subtracted, so every route the trim + will actually see stays in the assertion. + """ + env: Final = {**os.environ, "LITELLM_COMPONENT_ALLOWLIST_REPO_ROOT": _REPO_ROOT} + for key, value in _THROWAWAY_ENV.items(): + env.setdefault(key, value) + + probe: Final = run_child_interpreter(_COVERAGE_PROBE, env=env, timeout=90) + assert probe.returncode == 0, f"route probe failed:\n{probe.stderr}" + report: Final = json.loads(probe.stdout) + + assert not report["lazy_loaded"], ( + "route probe was not pristine; it loaded lazy features " + f"{report['lazy_loaded']}, so its route table is not the startup one" ) - backend_paths = _component_paths( - app.router.routes, BACKEND_EXACT_PATHS, BACKEND_PATH_PREFIXES + assert report["route_count"] > 100, ( + f"route probe only saw {report['route_count']} routes, so an empty " + "uncovered set would not mean anything" ) - uncovered = all_paths - (gateway_paths | backend_paths) - + uncovered: Final = report["uncovered"] assert not uncovered, ( f"{len(uncovered)} route(s) are not exposed on either component. " f"Update gateway/routes/allowlist.py or backend/routes/allowlist.py to cover:\n " - + "\n ".join(sorted(uncovered)) + + "\n ".join(uncovered) ) diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 9257a2dd23d..8fb53d4b0a0 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -34,6 +34,7 @@ from litellm.proxy.litellm_pre_call_utils import ( add_provider_specific_headers_to_request, check_if_token_is_service_account, clean_headers, + move_guardrails_to_metadata, ) from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY @@ -5187,6 +5188,45 @@ def test_clean_headers_strips_x_api_key_when_byok_enabled_but_x_api_key_was_auth # --------------------------------------------------------------------------- +@pytest.mark.asyncio +async def test_move_guardrails_to_metadata_moves_include_guardrail_response_before_the_no_guardrail_early_out(): + policy_registry = MagicMock() + policy_registry.is_initialized.return_value = False + user_api_key_dict = UserAPIKeyAuth(api_key="test-key") + + true_data = { + "model": "gpt-4.1-mini", + "messages": [{"role": "user", "content": "Hello"}], + "metadata": {}, + "include_guardrail_response": True, + } + with patch("litellm.proxy.policy_engine.policy_registry.get_policy_registry", return_value=policy_registry): + await move_guardrails_to_metadata( + data=true_data, + _metadata_variable_name="metadata", + user_api_key_dict=user_api_key_dict, + ) + + assert "include_guardrail_response" not in true_data + assert true_data["metadata"]["include_guardrail_response"] is True + + string_data = { + "model": "gpt-4.1-mini", + "messages": [{"role": "user", "content": "Hello"}], + "metadata": {}, + "include_guardrail_response": "true", + } + with patch("litellm.proxy.policy_engine.policy_registry.get_policy_registry", return_value=policy_registry): + await move_guardrails_to_metadata( + data=string_data, + _metadata_variable_name="metadata", + user_api_key_dict=user_api_key_dict, + ) + + assert "include_guardrail_response" not in string_data + assert string_data["metadata"]["include_guardrail_response"] is False + + @pytest.mark.asyncio async def test_team_guardrail_merges_with_global_policy(): """ @@ -8198,3 +8238,45 @@ def test_default_team_settings_bool_turn_off_message_logging_redacts(): ) is True ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/mcp-rest/tools/call", "/v1/responses", "/v1/chat/completions"]) +@pytest.mark.parametrize("custom_auth", ["x-mcp-auth", "x-private-mcp-token"]) +async def test_mcp_credentials_only_removed_from_logging_copies(path: str, custom_auth: str): + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + metadata_name: Final = "litellm_metadata" if path == "/v1/responses" else "metadata" + secrets: Final = { + "X-MCP-Deepwiki-Authorization": "upstream-sentinel", + custom_auth: "client-auth-sentinel", + "x-service-token": "configured-secret-sentinel", + } + attribution: Final = {"x-app-id": "app-a", "x-nuid": "user-a", "x-user-id": "identity-a"} + request: Final = _make_request_mock(path, {"Content-Type": "application/json", **secrets, **attribution}) + request.headers = Headers(request.headers) + settings: Final = {"mcp_client_side_auth_header_name": custom_auth, "user_header_name": "x-user-id"} + server: Final = MCPServer( + server_id="header-test", name="header-test", transport="http", url="https://example.com/mcp", + extra_headers=["x-service-token", "x-user-id"], + ) + with ( + patch("litellm.proxy.proxy_server.general_settings", settings), + patch.dict( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager.config_mcp_servers", + {"header-test": server}, clear=True, + ), + ): + updated: Final = await add_litellm_data_to_request( + data={"model": "test-model", "messages": [{"role": "user", "content": "hello"}]}, + request=request, user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), general_settings=settings, version="test", + ) + for header_dict in _all_header_dicts(updated, metadata_name): + assert not any(value in json.dumps(header_dict) for value in secrets.values()) + assert updated[metadata_name]["headers"] == updated["proxy_server_request"]["headers"] + for name, value in attribution.items(): + assert updated[metadata_name]["headers"][name] == value + for name, value in secrets.items(): + assert updated["secret_fields"]["raw_headers"][name.lower()] == value + assert request.headers[name] == value diff --git a/tests/test_litellm/proxy/test_native_compaction.py b/tests/test_litellm/proxy/test_native_compaction.py new file mode 100644 index 00000000000..778c1715530 --- /dev/null +++ b/tests/test_litellm/proxy/test_native_compaction.py @@ -0,0 +1,166 @@ +import asyncio +from collections.abc import Awaitable, Mapping +from types import MappingProxyType +from typing import Final, Literal + +import pytest +from fastapi import FastAPI, Request +from pydantic import TypeAdapter + +import litellm +from litellm.caching.caching import DualCache +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.redact_messages import should_redact_message_logging +from litellm.proxy import common_request_processing, proxy_server +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.auth_checks import can_key_call_model +from litellm.proxy._types import ProxyException +from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash, get_request_stash +from litellm.proxy.native_compaction import with_proxy_compaction_executor +from litellm.router import Router +from litellm.router_strategy.complexity_router.context_compaction import compaction_executor, reject_recursive_compactor +from litellm.types.utils import ModelResponse + +_HEADERS: Final = ( + (b"authorization", b"Bearer sk-compaction-fixture"), (b"cookie", b"session=fixture"), + (b"content-length", b"99999"), (b"x-litellm-call-id", b"parent"), + (b"litellm-disable-message-redaction", b"true"), (b"x-litellm-num-retries", b"8"), + (b"X-LiteLLM-Timeout", b"600"), (b"x-litellm-stream-timeout", b"500"), +) + + +async def _child( + protocol: Literal["chat", "messages"] = "chat", forged: bool = False, parent_model: str | None = None +) -> Mapping[str, object]: + executor: Final = compaction_executor.get() + assert executor is not None + payload: Final = TypeAdapter(Mapping[str, object]).validate_json( + b'{"model":"compactor","messages":[{"role":"user","content":"history"}],' + b'"num_retries":0,"timeout":7,"stream_timeout":7,"disable_fallbacks":true,"stream":false,' + b'"metadata":{"turn_off_message_logging":true}}' + ) + return await executor(protocol, MappingProxyType({ + "litellm_metadata" if protocol == "messages" and key == "metadata" else key: value + for key, value in payload.items() if forged or key != "metadata" + }), parent_model) + + +def _request(app: FastAPI) -> Request: + return Request(TypeAdapter(dict[str, object]).validate_python(MappingProxyType({ + "type": "http", "app": app, "scheme": "https", "server": ("proxy.test", 443), + "path": "/gateway/parent", "root_path": "/gateway", "query_string": b"parent=1", + "client": ("192.0.2.1", 4321), "headers": _HEADERS, + }))) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("protocol", ("chat", "messages")) +async def test_child_preserves_credentials_and_isolates_context(protocol: Literal["chat", "messages"]) -> None: + app: Final = FastAPI() + stash: Final = get_or_create_request_stash() + + @app.post("/v1/chat/completions" if protocol == "chat" else "/v1/messages") + async def endpoint(request: Request) -> Mapping[str, object]: + assert get_request_stash() is None and compaction_executor.get() is None + assert request.client == ("192.0.2.1", 4321) and request.url.scheme == "https" + assert request.scope["root_path"] == "/gateway" and request.cookies["session"] == "fixture" + assert request.headers["authorization"] == "Bearer sk-compaction-fixture" and not request.query_params + assert "x-litellm-call-id" not in request.headers + assert "litellm-disable-message-redaction" not in request.headers + assert int(request.headers["content-length"]) == len(await request.body()) + with pytest.raises(BadRequestError, match="regular model group"): + reject_recursive_compactor("auto-router") + return MappingProxyType({"summary": "compacted"}) + + with inherit_message_logging_privacy(True): + assert (await with_proxy_compaction_executor(_child(protocol), _request(app)))["summary"] == "compacted" + assert get_request_stash() is stash and compaction_executor.get() is None + reject_recursive_compactor("auto-router") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("protocol", ("chat", "messages")) +@pytest.mark.parametrize("policy", ("allowed", "denied", "forged", "router_alias", "unrelated_alias")) +async def test_real_proxy_child_auth_privacy_and_body_policy( + monkeypatch: pytest.MonkeyPatch, protocol: Literal["chat", "messages"], policy: str, +) -> None: + cache: Final = DualCache() + token: Final = proxy_server.hash_token("sk-compaction-fixture") + models: Final = {"denied": ("answer",), "router_alias": ("auto",), "unrelated_alias": ("other-auto",)}.get(policy, ("compactor",)) + auth: Final = UserAPIKeyAuth.model_validate(MappingProxyType({"token": token, "models": models})) + await cache.async_set_cache(key=token, value=auth) + dispatched: Final = asyncio.Event() + allowed: Final = policy in ("allowed", "router_alias") + + async def route( + data: Mapping[str, object], llm_router: Router | None, user_model: str | None, + route_type: str, user_api_key_dict: UserAPIKeyAuth | None, + ) -> Awaitable[ModelResponse]: + dispatched.set() + assert allowed + if policy == "router_alias": + with pytest.raises(ProxyException): + await can_key_call_model("unrelated-compactor", None, auth, None) + assert (data["num_retries"], data["timeout"], data["stream_timeout"]) == (0, 7, 7) + assert data["disable_fallbacks"] is True and data["stream"] is False + logging: Final = data["litellm_logging_obj"] + assert isinstance(logging, Logging) + assert logging.standard_callback_dynamic_params.get("turn_off_message_logging") is True + assert should_redact_message_logging(TypeAdapter(dict[str, object]).validate_python(MappingProxyType({ + "litellm_params": data, "standard_callback_dynamic_params": logging.standard_callback_dynamic_params, + }))) + return asyncio.sleep(0, result=ModelResponse(id="private-summary", model="compactor")) + + monkeypatch.setattr(litellm, "max_budget", 0) + monkeypatch.setattr(proxy_server.app, "dependency_overrides", {}) + monkeypatch.setattr(proxy_server, "master_key", "sk-master-fixture") + monkeypatch.setattr(litellm, "max_budget", 0.0) + monkeypatch.setattr(proxy_server, "prisma_client", object()) + monkeypatch.setattr(proxy_server, "user_api_key_cache", cache) + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(common_request_processing, "route_request", route) + with inherit_message_logging_privacy(True): + call: Final = with_proxy_compaction_executor( + _child(protocol, policy == "forged", "auto" if policy.endswith("alias") else None), _request(proxy_server.app) + ) + if allowed: + assert (await call)["id"] == "private-summary" + else: + status: Final = 401 if policy == "forged" else 403 + with pytest.raises(BadRequestError, match=rf"child request failed \(HTTP {status}\)"): + await call + assert dispatched.is_set() is allowed + assert compaction_executor.get() is None + if policy.endswith("alias"): + with pytest.raises(ProxyException): + await can_key_call_model("compactor", None, auth, None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("timeout", [False, True]) +async def test_cancelling_parent_cancels_and_drains_child(timeout: bool) -> None: + app: Final = FastAPI() + started: Final = asyncio.Event() + stopped: Final = asyncio.Event() + + @app.post("/v1/chat/completions") + async def endpoint() -> None: + started.set() + try: + await asyncio.Event().wait() + finally: + stopped.set() + + parent: Final = asyncio.create_task(with_proxy_compaction_executor(_child(), _request(app))) + await asyncio.wait_for(started.wait(), timeout=5) + if timeout: + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(parent, timeout=0) + else: + parent.cancel() + with pytest.raises(asyncio.CancelledError): + await parent + assert stopped.is_set() and compaction_executor.get() is None diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index a38470d1fdf..b84efda3308 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -10,6 +10,8 @@ import pytest import builtins +import runpy +import sys import types import urllib.parse as urlparse @@ -18,6 +20,7 @@ import yaml from uvicorn.config import LOOP_FACTORIES from uvicorn.importer import import_from_string +from litellm.proxy import proxy_cli from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server @@ -636,6 +639,36 @@ class TestProxyInitializationHelpers: ), f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() + @patch("uvicorn.run") + @patch("atexit.register") + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) + def test_script_boot_imports_the_package_proxy_server( + self, mock_should_update, mock_setup_db, mock_atexit_register, mock_uvicorn_run + ): + package_proxy_server = MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) + sibling_proxy_server = types.ModuleType("proxy_server") + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.dict( + "sys.modules", + {"proxy_server": sibling_proxy_server, "litellm.proxy.proxy_server": package_proxy_server}, + ), + patch.object(sys, "argv", ["proxy_cli.py", "--skip_server_startup"]), + patch.object(sys, "path", list(sys.path)), + pytest.raises(SystemExit) as exit_info, + ): + runpy.run_path(proxy_cli.__file__, run_name="__main__") + + assert exit_info.value.code == 0 + package_proxy_server.save_worker_config.assert_called_once() + @patch("uvicorn.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") @@ -1174,6 +1207,172 @@ class TestProxyInitializationHelpers: else: assert "pgbouncer" not in appended_params + @pytest.mark.parametrize( + "env_value, config_value, expect_pgbouncer", + [ + ("true", None, True), + ("1", None, True), + ("false", None, False), + (None, None, False), + ("true", False, True), + ("false", True, True), + ], + ) + @patch("subprocess.run") + @patch("atexit.register") + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + @patch( + "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False + ) + def test_disable_prepared_statements_env_var_forwarded_to_url( + self, + mock_should_update, + mock_setup_db, + mock_atexit_register, + mock_subprocess_run, + env_value, + config_value, + expect_pgbouncer, + ): + from click.testing import CliRunner + + from litellm.proxy.proxy_cli import run_server + + runner = CliRunner() + mock_subprocess_run.return_value = MagicMock(returncode=0) + + general_settings = {"database_url": "postgresql://test:test@localhost:5432/test"} + if config_value is not None: + general_settings["database_disable_prepared_statements"] = config_value + mock_proxy_module = MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) + mock_proxy_module.ProxyConfig.return_value.get_config = AsyncMock( + return_value={"general_settings": general_settings} + ) + + clean_env = { + k: v + for k, v in os.environ.items() + if k not in ("DATABASE_URL", "DIRECT_URL", "DATABASE_DISABLE_PREPARED_STATEMENTS") + } + if env_value is not None: + clean_env["DATABASE_DISABLE_PREPARED_STATEMENTS"] = env_value + + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.dict( + "sys.modules", + { + "proxy_server": mock_proxy_module, + "litellm.proxy.proxy_server": mock_proxy_module, + }, + ), + patch( + "litellm.proxy.proxy_cli.ProxyInitializationHelpers._get_default_unvicorn_init_args" + ) as mock_get_args, + patch( + "litellm.proxy.proxy_cli.append_query_params", + side_effect=lambda url, params: str(url), + ) as mock_append_query_params, + ): + mock_get_args.return_value = { + "app": "litellm.proxy.proxy_server:app", + "host": "localhost", + "port": 8000, + } + + result = runner.invoke( + run_server, + ["--local", "--config", "test-config.yaml", "--skip_server_startup"], + ) + + assert ( + result.exit_code == 0 + ), f"exit_code={result.exit_code}, output={result.output}" + appended_params = mock_append_query_params.call_args.args[1] + if expect_pgbouncer: + assert appended_params["pgbouncer"] == "true", appended_params + else: + assert "pgbouncer" not in appended_params, appended_params + + @patch("subprocess.run") + @patch("atexit.register") + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + @patch( + "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False + ) + def test_malformed_disable_prepared_statements_env_var_is_rejected_even_when_config_enables_it( + self, + mock_should_update, + mock_setup_db, + mock_atexit_register, + mock_subprocess_run, + ): + from click.testing import CliRunner + + from litellm.proxy.proxy_cli import run_server + + runner = CliRunner() + mock_subprocess_run.return_value = MagicMock(returncode=0) + + mock_proxy_module = MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) + mock_proxy_module.ProxyConfig.return_value.get_config = AsyncMock( + return_value={ + "general_settings": { + "database_url": "postgresql://test:test@localhost:5432/test", + "database_disable_prepared_statements": True, + } + } + ) + + clean_env = { + k: v + for k, v in os.environ.items() + if k not in ("DATABASE_URL", "DIRECT_URL", "DATABASE_DISABLE_PREPARED_STATEMENTS") + } + clean_env["DATABASE_DISABLE_PREPARED_STATEMENTS"] = "enabled" + + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.dict( + "sys.modules", + { + "proxy_server": mock_proxy_module, + "litellm.proxy.proxy_server": mock_proxy_module, + }, + ), + patch( + "litellm.proxy.proxy_cli.ProxyInitializationHelpers._get_default_unvicorn_init_args" + ) as mock_get_args, + patch( + "litellm.proxy.proxy_cli.append_query_params", + side_effect=lambda url, params: str(url), + ) as mock_append_query_params, + ): + mock_get_args.return_value = { + "app": "litellm.proxy.proxy_server:app", + "host": "localhost", + "port": 8000, + } + + result = runner.invoke( + run_server, + ["--local", "--config", "test-config.yaml", "--skip_server_startup"], + ) + + assert isinstance(result.exception, ValueError), f"exit_code={result.exit_code}, output={result.output}" + assert "DATABASE_DISABLE_PREPARED_STATEMENTS" in str(result.exception), result.exception + mock_append_query_params.assert_not_called() + @patch("uvicorn.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") @@ -1779,13 +1978,18 @@ class TestProxyInitializationHelpers: mock_proxy_config_instance.get_config = mock_get_config mock_proxy_config.return_value = mock_proxy_config_instance - mock_proxy_server_module = MagicMock(app=mock_app) + mock_proxy_server_module = MagicMock( + app=mock_app, + ProxyConfig=mock_proxy_config, + KeyManagementSettings=mock_key_mgmt, + save_worker_config=mock_save_worker_config, + ) # Only remove DATABASE_URL and DIRECT_URL to prevent the database setup # code path from running. Do NOT use clear=True as it removes PATH, HOME, # etc., which causes imports inside run_server to break in CI (the real - # litellm.proxy.proxy_server import at line 820 of proxy_cli.py has heavy - # side effects that fail without a proper environment). + # litellm.proxy.proxy_server import has heavy side effects that fail + # without a proper environment). env_overrides = { "DATABASE_URL": "", "DIRECT_URL": "", @@ -1801,18 +2005,7 @@ class TestProxyInitializationHelpers: with ( patch.dict( "sys.modules", - { - "proxy_server": MagicMock( - app=mock_app, - ProxyConfig=mock_proxy_config, - KeyManagementSettings=mock_key_mgmt, - save_worker_config=mock_save_worker_config, - ), - # Also mock litellm.proxy.proxy_server to prevent the real - # import at line 820 of proxy_cli.py which has heavy side - # effects (FastAPI app init, logging setup, etc.) - "litellm.proxy.proxy_server": mock_proxy_server_module, - }, + {"litellm.proxy.proxy_server": mock_proxy_server_module}, ), patch( "litellm.proxy.proxy_cli.ProxyInitializationHelpers._get_default_unvicorn_init_args" diff --git a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py index dd330d32ce6..9eb89b2f301 100644 --- a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py +++ b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py @@ -347,6 +347,60 @@ async def test_post_call_stream_guardrail_keeps_own_iterator_on_chat_completions assert delivered_text != "" +@pytest.mark.asyncio +async def test_post_call_stream_records_masked_text_for_deferred_logging(monkeypatch): + from litellm.caching.caching import DualCache + from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + monkeypatch.setattr(litellm, "callbacks", [_content_filter_guardrail("MASK")]) + proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) + logging_obj = _streaming_logging_obj() + + async def fake_stream(): + yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="the zebra runs"))]) + yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")]) + + delivered_text = "" + async for chunk in proxy_logging.async_post_call_streaming_iterator_hook( + response=fake_stream(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/chat/completions"), + request_data={"model": "gpt-4o-mini", "metadata": {}, "litellm_logging_obj": logging_obj}, + ): + for choice in chunk.choices: + delivered_text += choice.delta.content or "" + + assert "zebra" not in delivered_text + assert logging_obj.model_call_details[SERVED_OUTPUT_TEXTS_KEY] == (delivered_text,) + + +@pytest.mark.asyncio +async def test_post_call_stream_records_the_served_text_when_the_client_disconnects(monkeypatch): + from litellm.caching.caching import DualCache + from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + monkeypatch.setattr(litellm, "callbacks", [_content_filter_guardrail("MASK")]) + proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) + logging_obj = _streaming_logging_obj() + + async def fake_stream(): + yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="the zebra runs"))]) + yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content=" far"))]) + + stream = proxy_logging.async_post_call_streaming_iterator_hook( + response=fake_stream(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/chat/completions"), + request_data={"model": "gpt-4o-mini", "metadata": {}, "litellm_logging_obj": logging_obj}, + ) + first = await stream.__anext__() + await stream.aclose() + + delivered_text = "".join(choice.delta.content or "" for choice in first.choices) + assert "zebra" not in delivered_text + assert logging_obj.model_call_details[SERVED_OUTPUT_TEXTS_KEY] == (delivered_text,) + + @pytest.mark.asyncio async def test_unified_guardrail_iterator_accepts_explicit_guardrail(): """ @@ -489,6 +543,68 @@ async def test_post_call_stream_masking_guardrail_keeps_own_iterator_on_anthropi assert delivered == chunks +@pytest.mark.asyncio +async def test_post_call_stream_presidio_output_masking_masks_anthropic_messages_stream(monkeypatch): + """Regression: the presidio output-masking callback built by initialize_presidio + was rerouted onto the unified scan-only path on /v1/messages, so a card number + the analyzer flagged still streamed to the caller unmasked.""" + import json + + from litellm.caching.caching import DualCache + from litellm.proxy.guardrails.guardrail_registry import InMemoryGuardrailHandler + from litellm.types.guardrails import SupportedGuardrailIntegrations + + handler = InMemoryGuardrailHandler() + result = handler.initialize_guardrail( + guardrail={ + "guardrail_name": "presidio-card-mask", + "litellm_params": { + "guardrail": SupportedGuardrailIntegrations.PRESIDIO.value, + "mode": ["pre_call", "post_call"], + "default_on": True, + "presidio_analyzer_api_base": "https://fakelink.com/v1/presidio/analyze", + "presidio_anonymizer_api_base": "https://fakelink.com/v1/presidio/anonymize", + "pii_entities_config": {"CREDIT_CARD": "MASK"}, + "mock_redacted_text": {"text": "", "items": []}, + }, + } + ) + guardrail_id = result["guardrail_id"] + callbacks = [ + handler.guardrail_id_to_custom_guardrail[guardrail_id], + *handler.guardrail_id_to_sibling_callbacks[guardrail_id], + ] + monkeypatch.setattr(litellm, "callbacks", callbacks) + + chunks = _anthropic_stream_chunks(["4111", " 1111 1111 1111"]) + + async def fake_stream(): + for chunk in chunks: + yield chunk + + delivered = [] + async for chunk in ProxyLogging(user_api_key_cache=DualCache()).async_post_call_streaming_iterator_hook( + response=fake_stream(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/messages"), + request_data={ + "model": "claude-sonnet-5", + "litellm_logging_obj": _streaming_logging_obj(), + "metadata": {}, + }, + ): + delivered.append(chunk) + + wire = b"".join(delivered).decode() + text_deltas = [ + json.loads(line[6:])["delta"]["text"] + for line in wire.split("\n") + if line.startswith("data: ") and json.loads(line[6:]).get("delta", {}).get("type") == "text_delta" + ] + assert "4111" not in wire, wire + assert "".join(text_deltas) == "", wire + assert wire.count("event: message_stop") == 1, wire + + class _AppliesGuardrail(CustomGuardrail): """Implements the unified interface only, so the proxy routes it to unified_guardrail.""" diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 2792e176e0e..89fd9c5c9d4 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -3200,7 +3200,7 @@ def test_normalize_datetime_for_sorting(): @pytest.mark.asyncio -async def test_add_proxy_budget_to_db_only_creates_user_no_keys(): +async def test_add_proxy_budget_to_db_only_creates_user_no_keys(monkeypatch: pytest.MonkeyPatch): """ Test that _add_proxy_budget_to_db only creates a user and no keys are added. @@ -3218,8 +3218,8 @@ async def test_add_proxy_budget_to_db_only_creates_user_no_keys(): from litellm.proxy.proxy_server import ProxyStartupEvent # Set up required litellm settings - litellm.budget_duration = "30d" - litellm.max_budget = 100.0 + monkeypatch.setattr(litellm, "budget_duration", "30d") + monkeypatch.setattr(litellm, "max_budget", 100.0) litellm_proxy_budget_name = "litellm-proxy-budget" @@ -3258,7 +3258,7 @@ async def test_add_proxy_budget_to_db_only_creates_user_no_keys(): @pytest.mark.asyncio -async def test_add_proxy_budget_to_db_backfills_budget_reset_at(): +async def test_add_proxy_budget_to_db_backfills_budget_reset_at(monkeypatch: pytest.MonkeyPatch): """ Test that _upsert_proxy_budget_with_reset_at_backfill issues a conditional update_many with `WHERE budget_reset_at IS NULL` to backfill the column on @@ -3276,8 +3276,8 @@ async def test_add_proxy_budget_to_db_backfills_budget_reset_at(): import litellm from litellm.proxy.proxy_server import ProxyStartupEvent - litellm.budget_duration = "30d" - litellm.max_budget = 100.0 + monkeypatch.setattr(litellm, "budget_duration", "30d") + monkeypatch.setattr(litellm, "max_budget", 100.0) litellm_proxy_budget_name = "litellm-proxy-budget" mock_prisma = MagicMock() @@ -3555,6 +3555,23 @@ async def test_load_config_rejects_malformed_role_permissions(tmp_path): await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file)) +@pytest.mark.asyncio +async def test_load_config_compiles_key_alias_pattern_at_startup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.proxy_server import ProxyConfig + + monkeypatch.setattr(litellm, "key_alias_pattern", None) + config_file: Final = tmp_path / "config.yaml" + + config_file.write_text(yaml.dump({"model_list": [], "litellm_settings": {"key_alias_pattern": "^team-("}})) + with pytest.raises(Exception, match=r"litellm_settings\.key_alias_pattern"): + await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file)) + assert litellm.key_alias_pattern is None + + config_file.write_text(yaml.dump({"model_list": [], "litellm_settings": {"key_alias_pattern": "^team-[a-z]+$"}})) + await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file)) + assert litellm.key_alias_pattern == "^team-[a-z]+$" + + def test_os_environ_resolution_leaves_the_config_layer_holding_the_reference(monkeypatch): from litellm.proxy.proxy_server import ProxyConfig @@ -14877,7 +14894,7 @@ async def test_token_counter_keeps_the_event_loop_free_during_a_huggingface_coun async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeypatch): - from tokenizers import Tokenizer + from litellm.rust_bridge._native import Tokenizer from litellm import Router from tests.test_litellm.litellm_core_utils.event_loop_lag import assert_loop_stayed_free, timed_with_loop_lags @@ -14890,7 +14907,7 @@ async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeyp time.sleep(0.3) return claude_tokenizer - monkeypatch.setattr(litellm.utils, "Tokenizer", SlowHubTokenizer) + monkeypatch.setattr("litellm.rust_bridge.tokenizer.from_pretrained", SlowHubTokenizer.from_pretrained) monkeypatch.setattr( "litellm.proxy.proxy_server.llm_router", Router( @@ -14914,7 +14931,7 @@ async def test_token_counter_loads_a_custom_tokenizer_off_the_event_loop(monkeyp async def test_token_counter_loads_a_custom_tokenizer_once_per_identifier_revision_and_token(monkeypatch): - from tokenizers import Tokenizer + from litellm.rust_bridge._native import Tokenizer from litellm import Router from litellm.types.router import DeploymentTypedDict @@ -14931,7 +14948,7 @@ async def test_token_counter_loads_a_custom_tokenizer_once_per_identifier_revisi }, } - monkeypatch.setattr(litellm.utils, "Tokenizer", MagicMock(from_pretrained=from_pretrained)) + monkeypatch.setattr("litellm.rust_bridge.tokenizer.from_pretrained", from_pretrained) monkeypatch.setattr( "litellm.proxy.proxy_server.llm_router", Router( diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 734408d9b61..0fc7295a717 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -6,10 +6,12 @@ import pytest from fastapi import HTTPException from litellm.caching.caching import DualCache +from litellm.exceptions import InternalServerError from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.bug_report import ISSUE_URL_BASE from litellm.proxy._types import ProxyErrorTypes, UserAPIKeyAuth -from litellm.proxy.utils import PrismaClient, ProxyLogging +from litellm.proxy.utils import PrismaClient, ProxyLogging, handle_exception_on_proxy from litellm.types.guardrails import GuardrailEventHooks @@ -2418,3 +2420,16 @@ def test_mcp_auth_policy_uses_original_request_model(monkeypatch, model, expecte synthetic = proxy_logging._convert_mcp_to_llm_format(proxy_logging._create_mcp_request_object_from_kwargs(kwargs), kwargs) assert ("model-rule" in synthetic["metadata"]["guardrails"]) is expected assert "request-rule" in synthetic["metadata"]["guardrails"] + + +def test_handle_exception_on_proxy_logs_bug_report_only_for_unmapped_500(caplog): + with caplog.at_level("ERROR", logger="LiteLLM Proxy"): + provider_result = handle_exception_on_proxy( + InternalServerError(message="upstream 500", llm_provider="openai", model="gpt-4") + ) + assert ISSUE_URL_BASE not in caplog.text + internal_result = handle_exception_on_proxy(KeyError("missing")) + + assert provider_result.code == internal_result.code == "500" + assert ISSUE_URL_BASE in caplog.text + assert ISSUE_URL_BASE not in internal_result.message diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index 6cbbc279748..0b51062dd66 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -1282,7 +1282,7 @@ async def test_route_request_routing_group_name_passes_model_gate(): @pytest.mark.asyncio -async def test_route_request_a2a_agent_miss_does_not_consume_model_read_through(monkeypatch): +async def test_route_request_a2a_agent_miss_does_not_consume_model_read_through(fresh_agent_read_through, monkeypatch): from types import SimpleNamespace from unittest.mock import AsyncMock diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index eecee2fd0f1..0140fcaba21 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -1,5 +1,6 @@ import json import os +from typing import Final import pytest from fastapi.testclient import TestClient @@ -90,6 +91,16 @@ def mock_auth(): app.dependency_overrides.pop(user_api_key_auth, None) +@pytest.fixture(autouse=True) +def fresh_settings_store(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.config_resolvers.settings_store import SettingsStore + + store: Final = SettingsStore("general_settings") + monkeypatch.setattr(proxy_server.proxy_config, "settings", store) + monkeypatch.setattr(proxy_server, "general_settings", store) + + class TestProxySettingEndpoints: def test_get_internal_user_settings(self, mock_proxy_config, mock_auth): """Test getting the internal user settings""" @@ -3670,6 +3681,91 @@ class TestPtuCostAttributionUISetting: assert not mock_prisma.db.litellm_uisettings.upsert.called +class TestApplyUserBudgetToTeamKeysUISetting: + """``apply_user_budget_to_team_keys`` mirrors general_settings on every GET. + + The proxy enforces the key owner's user budget on team keys only when + ``general_settings.apply_user_budget_to_team_keys`` is on, so the dashboard + shows the owner's budget gate on a team key iff this derived value is true. + Like the other derived settings it is read-only and never persisted. + """ + + @staticmethod + def _mock_prisma(monkeypatch, stored=None): + from unittest.mock import AsyncMock, MagicMock + + mock_prisma = MagicMock() + mock_record = None + if stored is not None: + mock_record = MagicMock() + mock_record.ui_settings = stored + mock_prisma.db.litellm_uisettings.find_unique = AsyncMock(return_value=mock_record) + mock_prisma.db.litellm_uisettings.upsert = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + return mock_prisma + + def test_reported_false_when_general_settings_lacks_the_flag(self, mock_auth, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + self._mock_prisma(monkeypatch) + + response = client.get("/get/ui_settings") + + assert response.status_code == 200 + assert response.json()["values"]["apply_user_budget_to_team_keys"] is False + + def test_reported_true_when_general_settings_enables_the_flag(self, mock_auth, monkeypatch): + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"apply_user_budget_to_team_keys": True}, + ) + self._mock_prisma(monkeypatch) + + response = client.get("/get/ui_settings") + + assert response.status_code == 200 + assert response.json()["values"]["apply_user_budget_to_team_keys"] is True + + def test_non_json_values_elsewhere_in_general_settings_do_not_break_get(self, mock_auth, monkeypatch): + """general_settings holds non-JSON values at runtime (e.g. RoleBasedPermissions + instances under "role_permissions"); only the flag itself may be inspected.""" + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {"role_permissions": [object()], "apply_user_budget_to_team_keys": True}, + ) + self._mock_prisma(monkeypatch) + + response = client.get("/get/ui_settings") + + assert response.status_code == 200 + assert response.json()["values"]["apply_user_budget_to_team_keys"] is True + + def test_reads_only_the_flag_key(self): + from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( + _apply_user_budget_to_team_keys_enabled, + ) + + assert _apply_user_budget_to_team_keys_enabled({"apply_user_budget_to_team_keys": True}) is True + assert _apply_user_budget_to_team_keys_enabled({}) is False + assert _apply_user_budget_to_team_keys_enabled({"apply_user_budget_to_team_keys": "true"}) is False + + def test_a_persisted_true_cannot_forge_the_derived_value(self, mock_auth, monkeypatch): + """A row written before the allowlist existed must not be able to turn the feature on.""" + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + self._mock_prisma(monkeypatch, stored={"apply_user_budget_to_team_keys": True}) + + response = client.get("/get/ui_settings") + + assert response.status_code == 200 + assert response.json()["values"]["apply_user_budget_to_team_keys"] is False + + def test_is_not_an_allowlisted_persisted_setting(self): + from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( + ALLOWED_UI_SETTINGS_FIELDS, + ) + + assert "apply_user_budget_to_team_keys" not in ALLOWED_UI_SETTINGS_FIELDS + + class TestTeamAdminEditableTeamFieldsSetting: """team_admin_editable_team_fields: the proxy-wide allow-list update_team applies to team admins.""" @@ -3796,6 +3892,31 @@ class TestTeamAdminEditableTeamFieldsSetting: assert "tpm_limit" in field_schema["items"]["enum"] assert "projects" in field_schema["items"]["enum"] + @pytest.mark.parametrize(("stored", "patched"), [(["tpm_limit"], []), (["rpm_limit"], ["max_budget"])]) + def test_get_reports_its_own_db_row_whatever_an_earlier_test_patched(self, monkeypatch, stored, patched): + """A booted proxy keeps its runtime settings in one shared store. Each case PATCHes a list into + that store, so whichever case ran second used to read the other's list instead of its own DB row.""" + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy import proxy_server + + mock_prisma = self._as_proxy_admin(monkeypatch) + mock_db_record = MagicMock() + mock_db_record.ui_settings = {"team_admin_editable_team_fields": stored} + mock_prisma.db.litellm_uisettings.find_unique = AsyncMock(return_value=mock_db_record) + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + + try: + fetched = client.get("/get/ui_settings") + proxy_server._bind_general_settings_store(proxy_server.proxy_config.settings) + response = client.patch("/update/ui_settings", json={"team_admin_editable_team_fields": patched}) + finally: + app.dependency_overrides.clear() + + assert fetched.json()["values"]["team_admin_editable_team_fields"] == stored + assert response.status_code == 200 + assert proxy_server.general_settings["team_admin_editable_team_fields"] == patched + class TestSyncUiSettingsToGeneralSettings: """The DB re-read each pod runs on startup and on every config reload.""" @@ -3877,6 +3998,32 @@ class TestSyncUiSettingsToGeneralSettings: assert general_settings["forward_client_headers_to_llm_api"] is True assert general_settings.source("forward_client_headers_to_llm_api") == "db" + def test_every_runtime_flag_reaches_a_reader_once_applied(self, monkeypatch): + """A flag the settings rules do not route to the ui_settings row is stored but never read back.""" + from litellm.proxy import proxy_server + from litellm.proxy.config_resolvers import SettingsStore + from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( + TEAM_ADMIN_EDITABLE_TEAM_FIELDS_SETTING, + _RUNTIME_GENERAL_SETTINGS_FLAGS, + apply_runtime_general_settings_flags, + ) + + general_settings = SettingsStore("general_settings") + general_settings.load_yaml({}) + monkeypatch.setattr(proxy_server, "general_settings", general_settings) + + stored = { + key: (["tpm_limit"] if key == TEAM_ADMIN_EDITABLE_TEAM_FIELDS_SETTING else True) + for key in _RUNTIME_GENERAL_SETTINGS_FLAGS + } + assert stored + + apply_runtime_general_settings_flags(stored) + + read_back = {key: general_settings.get(key) for key in stored} + + assert read_back == stored + def test_applied_runtime_flags_cannot_override_the_config_file(self, monkeypatch): from litellm.proxy import proxy_server from litellm.proxy.config_resolvers import SettingsStore @@ -3890,3 +4037,4 @@ class TestSyncUiSettingsToGeneralSettings: assert general_settings["forward_client_headers_to_llm_api"] is False assert general_settings.source("forward_client_headers_to_llm_api") == "config" + diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py b/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py index a4bb7d63548..d7c52c528eb 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py @@ -5,16 +5,18 @@ from __future__ import annotations import asyncio from datetime import datetime -from typing import Any, Final +from types import MappingProxyType +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import HTTPException import litellm +from litellm.constants import PROXY_REJECTED_BEFORE_ROUTING_KEY from litellm.exceptions import GuardrailRaisedException from litellm.integrations.custom_logger import CustomLogger -from litellm.proxy._types import AlertType, ProxyErrorTypes, UserAPIKeyAuth +from litellm.proxy._types import ProxyErrorTypes, UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging @@ -99,6 +101,456 @@ async def test_post_call_failure_hook_no_callbacks_returns_none( } +@pytest.mark.asyncio +async def test_post_call_failure_hook_attributes_single_router_deployment( + proxy_logging, make_user_api_key_auth, monkeypatch +): + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + "model_info": {"provider": "acme"}, + } + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=HTTPException(status_code=403, detail="blocked"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + kwargs = recorded[0] + assert kwargs["custom_llm_provider"] == "openai" + assert kwargs["litellm_params"]["custom_llm_provider"] == "openai" + assert kwargs["litellm_params"]["metadata"]["model_info"]["provider"] == "acme" + assert kwargs["litellm_params"]["metadata"]["deployment"] == "openai/gpt-4.1" + assert kwargs["litellm_params"][PROXY_REJECTED_BEFORE_ROUTING_KEY] is True + assert kwargs["standard_logging_object"]["custom_llm_provider"] == "openai" + assert ( + kwargs["standard_logging_object"]["model_id"] == proxy_server.llm_router.get_model_list()[0]["model_info"]["id"] + ) + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_keeps_router_stamped_metadata_for_post_call_failures( + proxy_logging, make_user_api_key_auth, monkeypatch +): + """A post-call guardrail block arrives after the provider handoff with the router's own + ``model_info`` in the request metadata. The pre-routing flag must stay off so deployment + metrics keep attributing the failure to the deployment that actually served the call.""" + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + "model_info": {"id": "routed-deployment"}, + } + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + request_data = { + "litellm_call_id": "post-call-guardrail", + "model": "internal-model", + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"model_info": {"id": "routed-deployment", "served": True}}, + } + logging_obj, request_data = litellm.utils.function_setup( + original_function="acompletion", rules_obj=litellm.utils.Rules(), start_time=datetime.now(), **request_data + ) + logging_obj.model_call_details["first_api_call_start_time"] = datetime.now() + request_data["litellm_logging_obj"] = logging_obj + + await proxy_logging.post_call_failure_hook( + request_data=request_data, + original_exception=GuardrailRaisedException(guardrail_name="g", message="response blocked"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + kwargs = recorded[0] + assert kwargs["litellm_params"]["metadata"]["model_info"] == {"id": "routed-deployment", "served": True} + assert PROXY_REJECTED_BEFORE_ROUTING_KEY not in kwargs["litellm_params"] + assert kwargs["standard_logging_object"]["model_id"] == "routed-deployment" + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_flags_pre_routing_reject_despite_caller_model_info( + proxy_logging, make_user_api_key_auth, monkeypatch +): + """A key allowed to override pricing keeps caller-supplied ``metadata.model_info``. A reject + before any provider handoff must still carry the pre-routing flag so deployment metrics do + not record an outage for a deployment the request never reached.""" + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + "model_info": {"id": "real-deployment"}, + } + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={ + "model": "internal-model", + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"model_info": {"id": "spoofed-deployment"}}, + }, + original_exception=HTTPException(status_code=429, detail="key over limit"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + kwargs = recorded[0] + assert kwargs["litellm_params"][PROXY_REJECTED_BEFORE_ROUTING_KEY] is True + assert kwargs["litellm_params"]["custom_llm_provider"] == "openai" + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_attribution_does_not_count_against_the_deployment( + proxy_logging, make_user_api_key_auth, monkeypatch +): + """The router's failure callbacks run on this path too. A proxy-side reject must not + bump the deployment's failure or rpm counters, or a key hitting its own limit + could cool down the only deployment for everyone.""" + from litellm.proxy import proxy_server + + router = litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test", "rpm": 100}, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + proxy_logging.alert_types = [] + deployment_id = router.get_model_list()[0]["model_info"]["id"] + + for status in (403, 429): + await proxy_logging.post_call_failure_hook( + request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=HTTPException(status_code=status, detail="blocked"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + pending = asyncio.all_tasks() - {asyncio.current_task()} + await asyncio.gather(*pending, return_exceptions=True) + + deployment_keys = [key for key in router.cache.in_memory_cache.cache_dict if deployment_id in key] + assert deployment_keys == [], f"proxy reject was counted against the deployment: {deployment_keys}" + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_attributes_the_keys_team_deployment_over_the_global_group( + proxy_logging, make_user_api_key_auth, monkeypatch +): + """A team key requesting its team public model name must be attributed to the team's + deployment, not to a global group that happens to share the public name.""" + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "shared-name", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + "model_info": {"id": "global-deployment"}, + }, + { + "model_name": "shared-name_test-team_deadbeef", + "litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-test"}, + "model_info": { + "id": "team-deployment", + "team_id": "test-team", + "team_public_model_name": "shared-name", + }, + }, + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={"model": "shared-name", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=HTTPException(status_code=429, detail="rate limited"), + user_api_key_dict=make_user_api_key_auth(team_id="test-team", request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + kwargs = recorded[0] + assert kwargs["custom_llm_provider"] == "anthropic" + assert kwargs["litellm_params"]["metadata"]["deployment"] == "anthropic/claude-sonnet-4-5" + assert kwargs["standard_logging_object"]["model_id"] == "team-deployment" + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_omits_provider_for_mixed_router_deployments( + proxy_logging, make_user_api_key_auth, monkeypatch +): + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + }, + { + "model_name": "internal-model", + "litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": "sk-test"}, + }, + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=HTTPException(status_code=403, detail="blocked"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + kwargs = recorded[0] + assert kwargs.get("custom_llm_provider") is None + assert "model_info" not in (kwargs["litellm_params"].get("metadata") or {}) + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_omits_provider_when_a_deployment_does_not_resolve( + proxy_logging, make_user_api_key_auth, monkeypatch +): + """One deployment resolves to openai and its sibling resolves to nothing: the group + is not known to be single-provider, so no provider is stamped on the failure.""" + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + router = MagicMock() + router.get_model_list.return_value = [ + {"model_name": "internal-model", "litellm_params": {"model": "openai/gpt-4.1"}}, + {"model_name": "internal-model", "litellm_params": {"model": "unmapped-model-with-no-provider"}}, + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=HTTPException(status_code=403, detail="blocked"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + kwargs = recorded[0] + assert kwargs.get("custom_llm_provider") is None + assert kwargs["litellm_params"].get("custom_llm_provider") is None + + +@pytest.mark.asyncio +async def test_handle_logging_proxy_only_path_attributes_with_read_only_metadata( + proxy_logging, make_user_api_key_auth, monkeypatch +): + """With a logging object already on the request, its metadata is taken as given; + a read-only mapping there must not crash the stamp, and the failure handler + still receives the provider attribution.""" + from litellm.proxy import proxy_server + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + "model_info": {"provider": "acme"}, + } + ] + ), + ) + logging_obj = MagicMock() + logging_obj.call_type = "acompletion" + logging_obj.model_call_details = {} + logging_obj.async_failure_handler = AsyncMock() + + await proxy_logging._handle_logging_proxy_only_error( + request_data={ + "litellm_logging_obj": logging_obj, + "model": "internal-model", + "messages": [{"role": "user", "content": "hi"}], + "metadata": MappingProxyType({"user_api_key_alias": "frozen"}), + }, + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + original_exception=HTTPException(status_code=403, detail="blocked"), + ) + + assert logging_obj.async_failure_handler.called + update_kwargs = logging_obj.update_environment_variables.call_args.kwargs + assert update_kwargs["custom_llm_provider"] == "openai" + assert update_kwargs["litellm_params"]["custom_llm_provider"] == "openai" + assert update_kwargs["litellm_params"]["metadata"] == {"user_api_key_alias": "frozen"} + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_fires_without_router_attribution( + proxy_logging, make_user_api_key_auth, monkeypatch +): + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "different-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + } + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={"model": "internal-model", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=HTTPException(status_code=403, detail="blocked"), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + kwargs = recorded[0] + assert kwargs.get("custom_llm_provider") is None + assert "model_info" not in (kwargs["litellm_params"].get("metadata") or {}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", [123, ["internal-model"], {"name": "internal-model"}, None]) +async def test_post_call_failure_hook_fires_for_non_string_model( + proxy_logging, make_user_api_key_auth, monkeypatch, model: object +): + """A body whose ``model`` is not a string is rejected by the proxy before routing; its + failure callback must still fire, unattributed, instead of a TypeError escaping the hook.""" + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "internal-model", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + } + ] + ), + ) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={"model": model, "messages": [{"role": "user", "content": "hi"}]}, + original_exception=HTTPException(status_code=400, detail="'model' must be a string."), + user_api_key_dict=make_user_api_key_auth(request_route="/chat/completions"), + route="/chat/completions", + ) + + assert len(recorded) == 1 + kwargs = recorded[0] + assert kwargs.get("custom_llm_provider") is None + assert "model_info" not in (kwargs["litellm_params"].get("metadata") or {}) + + @pytest.mark.asyncio async def test_post_call_failure_hook_callback_returns_http_exception( proxy_logging, make_user_api_key_auth, monkeypatch diff --git a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py index 6e049d7634c..54e98cc2b6c 100644 --- a/tests/test_litellm/responses/mcp/test_chat_completions_handler.py +++ b/tests/test_litellm/responses/mcp/test_chat_completions_handler.py @@ -159,6 +159,7 @@ async def test_acompletion_with_mcp_passes_mcp_server_auth_headers_to_process_to secret_fields=secret_fields, ) + assert captured_process_kwargs["raw_headers"] == secret_fields["raw_headers"] assert "mcp_server_auth_headers" in captured_process_kwargs mcp_server_auth_headers = captured_process_kwargs["mcp_server_auth_headers"] assert mcp_server_auth_headers is not None diff --git a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py index 83537c236a3..57cebf489a2 100644 --- a/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/test_litellm/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -1,13 +1,15 @@ +import importlib import subprocess import sys import textwrap import types +from typing import Any, Final, cast from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import HTTPException +from mcp.types import CallToolResult, TextContent from openai.types.responses.tool_param import Mcp -import importlib from litellm.proxy._experimental.mcp_server.faults.list_outcomes import AggregateToolListing from litellm.responses import main as responses_main @@ -15,10 +17,9 @@ from litellm.responses.mcp import litellm_proxy_mcp_handler as mcp_handler_modul from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) -from typing import Any, cast from litellm.types.llms.openai import ResponsesAPIResponse -from litellm.types.utils import ModelResponse from litellm.types.responses.main import OutputFunctionToolCall +from litellm.types.utils import ModelResponse class _DummyMCPResult: @@ -492,6 +493,157 @@ async def test_execute_tool_calls_threads_logging_obj_into_call_tool(monkeypatch assert call_tool_mock.await_args.kwargs["litellm_logging_obj"] is sentinel_logging_obj +@pytest.mark.asyncio +async def test_execute_tool_calls_applies_post_call_hook_content(monkeypatch): + proxy_module = types.SimpleNamespace(proxy_logging_obj=None) + monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", proxy_module) + + result = CallToolResult( + content=[TextContent(type="text", text="SECRET-1234")], + structuredContent={"result": "SECRET-1234"}, + isError=False, + ) + fake_manager = types.SimpleNamespace( + get_registry=MagicMock(return_value={}), + call_tool=AsyncMock(return_value=result), + _get_mcp_server_from_tool_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(return_value=None), + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + fake_manager, + ) + + logging_obj = MagicMock() + logging_obj.model_call_details = {} + logging_obj.async_post_mcp_tool_call_hook = AsyncMock(return_value=CallToolResult(content=[TextContent(type="text", text="[REDACTED]")], is_error=True)) + logging_obj.async_success_handler = AsyncMock() + handler_module = importlib.import_module("litellm.responses.mcp.litellm_proxy_mcp_handler") + monkeypatch.setattr(handler_module, "function_setup", lambda *_args, **_kwargs: (logging_obj, None)) + + tool_name = "deepwiki-read_wiki_structure" + results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map={tool_name: "deepwiki"}, + tool_calls=[{"id": "call-1", "function": {"name": tool_name, "arguments": "{}"}}], + user_api_key_auth=None, + ) + + assert results == [{"tool_call_id": "call-1", "result": "[REDACTED]", "name": tool_name}] + assert logging_obj.async_success_handler.await_args.kwargs["result"].content[0].text == "[REDACTED]" + assert logging_obj.async_success_handler.await_args.kwargs["result"].structured_content is None + + +@pytest.mark.asyncio +async def test_execute_tool_calls_returns_proxy_result_without_logging(monkeypatch): + result = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_mcp_call_hook = AsyncMock(side_effect=lambda response, **_: response) + monkeypatch.setitem( + sys.modules, "litellm.proxy.proxy_server", types.SimpleNamespace(proxy_logging_obj=proxy_logging_obj) + ) + + fake_manager = types.SimpleNamespace( + get_registry=MagicMock(return_value={}), + call_tool=AsyncMock(return_value=result), + _get_mcp_server_from_tool_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(return_value=None), + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + fake_manager, + ) + handler_module = importlib.import_module("litellm.responses.mcp.litellm_proxy_mcp_handler") + monkeypatch.setattr(handler_module, "function_setup", lambda *_args, **_kwargs: (None, None)) + + tool_name = "deepwiki-read_wiki_structure" + results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map={tool_name: "deepwiki"}, + tool_calls=[{"id": "call-1", "function": {"name": tool_name, "arguments": "{}"}}], + user_api_key_auth=None, + ) + + assert results == [{"tool_call_id": "call-1", "result": "ok", "name": tool_name}] + proxy_logging_obj.post_mcp_call_hook.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_execute_tool_calls_passes_logging_details_to_proxy_hook(monkeypatch): + result = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_mcp_call_hook = AsyncMock(side_effect=lambda response, **_: response) + monkeypatch.setitem( + sys.modules, "litellm.proxy.proxy_server", types.SimpleNamespace(proxy_logging_obj=proxy_logging_obj) + ) + + fake_manager = types.SimpleNamespace( + get_registry=MagicMock(return_value={}), + call_tool=AsyncMock(return_value=result), + _get_mcp_server_from_tool_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(return_value=None), + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + fake_manager, + ) + logging_obj = MagicMock() + logging_obj.model_call_details = {"request_id": "request-1"} + logging_obj.async_post_mcp_tool_call_hook = AsyncMock(return_value=result) + logging_obj.async_success_handler = AsyncMock() + handler_module = importlib.import_module("litellm.responses.mcp.litellm_proxy_mcp_handler") + monkeypatch.setattr(handler_module, "function_setup", lambda *_args, **_kwargs: (logging_obj, None)) + + tool_name = "deepwiki-read_wiki_structure" + results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map={tool_name: "deepwiki"}, + tool_calls=[{"id": "call-1", "function": {"name": tool_name, "arguments": "{}"}}], + user_api_key_auth=None, + ) + + assert results == [{"tool_call_id": "call-1", "result": "ok", "name": tool_name}] + assert proxy_logging_obj.post_mcp_call_hook.await_args.kwargs["request_data"] == logging_obj.model_call_details + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_stage", ["post_call_hook", "success_handler"]) +async def test_execute_tool_calls_continues_when_post_call_logging_fails(monkeypatch, failure_stage: str): + proxy_module = types.SimpleNamespace(proxy_logging_obj=None) + monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", proxy_module) + + result = CallToolResult(content=[TextContent(type="text", text="ok")], isError=False) + fake_manager = types.SimpleNamespace( + get_registry=MagicMock(return_value={}), + call_tool=AsyncMock(return_value=result), + _get_mcp_server_from_tool_name=MagicMock(return_value=None), + get_mcp_server_by_name=MagicMock(return_value=None), + ) + monkeypatch.setattr( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager", + fake_manager, + ) + + logging_obj = MagicMock() + logging_obj.model_call_details = {} + logging_obj.post_call = MagicMock() + logging_obj.async_post_mcp_tool_call_hook = AsyncMock( + side_effect=RuntimeError("hook failed") if failure_stage == "post_call_hook" else None, + return_value=result, + ) + logging_obj.async_success_handler = AsyncMock( + side_effect=RuntimeError("success logging failed") if failure_stage == "success_handler" else None + ) + handler_module = importlib.import_module("litellm.responses.mcp.litellm_proxy_mcp_handler") + monkeypatch.setattr(handler_module, "function_setup", lambda *_args, **_kwargs: (logging_obj, None)) + + tool_name = "deepwiki-read_wiki_structure" + results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map={tool_name: "deepwiki"}, + tool_calls=[{"id": "call-1", "function": {"name": tool_name, "arguments": "{}"}}], + user_api_key_auth=None, + ) + + assert results == [{"tool_call_id": "call-1", "result": "ok", "name": tool_name}] + + @pytest.mark.asyncio async def test_get_mcp_tools_from_manager_enables_list_tools_logging(monkeypatch): """ @@ -1074,6 +1226,7 @@ async def test_mcp_follow_up_call_is_stateless_when_store_is_false( ) async def fake_process(**kwargs: Any) -> tuple[list[Any], dict[str, str]]: + assert kwargs["raw_headers"] == {"x-app-id": "follow-up-caller"} return ([], {"foo": "litellm_proxy"}) async def fake_execute(**kwargs: Any) -> list[dict[str, Any]]: @@ -1093,6 +1246,7 @@ async def test_mcp_follow_up_call_is_stateless_when_store_is_false( model="gpt-5", tools=[{"type": "mcp", "server_url": "litellm_proxy", "require_approval": "never"}], litellm_metadata={"guardrails": ["block-all"]}, + secret_fields={"raw_headers": {"x-app-id": "follow-up-caller"}}, store=store, previous_response_id=caller_previous_response_id, ) @@ -1105,3 +1259,39 @@ async def test_mcp_follow_up_call_is_stateless_when_store_is_false( item for item in follow_up_call["input"] if isinstance(item, dict) and item.get("type") == "reasoning" ] assert bool(reasoning_items) is (store is False) + + +@pytest.mark.asyncio +async def test_responses_discovery_logs_sanitized_caller_headers(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy._experimental.mcp_server import operations + from litellm.proxy._experimental.mcp_server import mcp_server_manager + + headers: Final = { + "x-app-id": "app-a", "x-nuid": "user-a", "x-user-id": "identity-a", + "x-mcp-deepwiki-authorization": "upstream-sentinel", "authorization": "proxy-sentinel", + } + manager: Final = types.SimpleNamespace( + get_registry=MagicMock(return_value={}), + get_allowed_mcp_servers=AsyncMock(return_value=[]), + get_mcp_servers_from_ids=MagicMock(return_value=[]), + ) + logger: Final = MagicMock(model_call_details={}) + logger.async_success_handler = AsyncMock() + setup: Final = MagicMock(return_value=(logger, None)) + monkeypatch.setattr(mcp_server_manager, "global_mcp_server_manager", manager) + monkeypatch.setattr(operations, "_get_allowed_mcp_servers", AsyncMock(return_value=[])) + monkeypatch.setattr(operations, "function_setup", setup) + response: Final = ResponsesAPIResponse( + id="resp_test", created_at=1234567891, model="test-model", object="response", + status="completed", output=[], parallel_tool_calls=False, tool_choice="auto", tools=[], + ) + monkeypatch.setattr(responses_main, "aresponses", AsyncMock(return_value=response)) + result: Final = await responses_main.aresponses_api_with_mcp( + input="hi", model="test-model", tools=[{"type": "mcp", "server_url": "litellm_proxy"}], + secret_fields={"raw_headers": headers}, + ) + assert result is response + logger.async_success_handler.assert_awaited_once() + logged: Final = setup.call_args.kwargs["metadata"]["headers"] + assert logged == {"x-app-id": "app-a", "x-nuid": "user-a", "x-user-id": "identity-a"} + assert headers["x-mcp-deepwiki-authorization"] == "upstream-sentinel" diff --git a/tests/test_litellm/responses/test_dispatch.py b/tests/test_litellm/responses/test_dispatch.py index 2990360d550..45cb5c4f1ad 100644 --- a/tests/test_litellm/responses/test_dispatch.py +++ b/tests/test_litellm/responses/test_dispatch.py @@ -13,7 +13,7 @@ from litellm.responses.dispatch import ( ) from litellm.rust_bridge import catalog from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Route, Rule +from litellm.rust_bridge.catalog import Route, RouteRule from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.responses.entrypoints import ( NATIVE_ARESPONSES, @@ -26,7 +26,7 @@ from litellm.types.llms.openai import ResponsesAPIResponse INPUT: Final = [{"role": "user", "content": "hi"}] PYTHON_RULES: Final = () -RUST_RULES: Final = (Rule(Route.RESPONSES, Rollout.RUST_REQUIRED),) +RUST_RULES: Final = (RouteRule(Route.RESPONSES, Rollout.RUST_REQUIRED),) def _response(model: str = "gpt-4o") -> ResponsesAPIResponse: @@ -102,7 +102,8 @@ async def test_async_python_route_forwards_original_call_shape() -> None: response: Final = _response() async def python( - *call_args: object, **call_kwargs: object # kwargs-ok: records call shape + *call_args: object, + **call_kwargs: object, # kwargs-ok: records call shape ) -> ResponsesAPIResponse: captured.append((call_args, call_kwargs)) return response @@ -143,9 +144,7 @@ def test_native_receives_normalized_request_and_original_call_shape() -> None: "custom_llm_provider": "anthropic", "litellm_metadata": metadata, } - captured: Final[ - list[tuple[LiteLLMResponsesRequest, tuple[object, ...], Mapping[str, object]]] - ] = [] + captured: Final[list[tuple[LiteLLMResponsesRequest, tuple[object, ...], Mapping[str, object]]]] = [] response: Final = _response("anthropic/claude-sonnet-4-5") def python(*call_args: object, **call_kwargs: object) -> ResponsesAPIResponse: # kwargs-ok: rejected fallback @@ -228,9 +227,7 @@ def test_internal_async_marker_bypasses_native() -> None: ((), {}), ), ) -def test_binding_errors_delegate_unchanged_to_python( - args: tuple[object, ...], kwargs: Mapping[str, object] -) -> None: +def test_binding_errors_delegate_unchanged_to_python(args: tuple[object, ...], kwargs: Mapping[str, object]) -> None: captured: Final[list[tuple[tuple[object, ...], Mapping[str, object]]]] = [] response: Final = _response() diff --git a/tests/test_litellm/responses/test_metadata_codex_callback.py b/tests/test_litellm/responses/test_metadata_codex_callback.py index f151f36be63..d5c411d63db 100644 --- a/tests/test_litellm/responses/test_metadata_codex_callback.py +++ b/tests/test_litellm/responses/test_metadata_codex_callback.py @@ -48,72 +48,6 @@ class MetadataCaptureCallback(CustomLogger): self.event.set() -@pytest.mark.asyncio -async def test_metadata_passed_to_custom_callback_codex_models(): - """ - Test that metadata passed to completion() is available in custom callback - when using codex models (responses API bridge path). - - Codex models have mode=responses and route through responses_api_bridge, - which passes litellm_metadata. The fix ensures this is preserved as - litellm_params.metadata for callback compatibility. - """ - from litellm.types.llms.openai import ResponsesAPIResponse - - mock_response = ResponsesAPIResponse.model_construct( - id="resp-test", - created_at=0, - output=[ - { - "type": "message", - "id": "msg-1", - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello!"}], - } - ], - object="response", - model="gpt-5.1-codex", - status="completed", - usage={ - "input_tokens": 5, - "output_tokens": 10, - "total_tokens": 15, - }, - ) - - test_metadata = {"foo": "bar", "trace_id": "test-123"} - callback = MetadataCaptureCallback() - original_callbacks = litellm.callbacks.copy() if litellm.callbacks else [] - litellm.callbacks = [callback] - - try: - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - new_callable=AsyncMock, - ) as mock_post: - mock_post.return_value = _make_mock_http_response( - mock_response.model_dump() - ) - # gpt-5.1-codex has mode=responses - routes through responses bridge - await litellm.acompletion( - model="gpt-5.1-codex", - messages=[{"role": "user", "content": "Hello"}], - metadata=test_metadata, - ) - - await asyncio.wait_for(callback.event.wait(), timeout=5.0) - - assert callback.captured_kwargs is not None, "Callback should have been invoked" - - litellm_params = callback.captured_kwargs.get("litellm_params", {}) - metadata = litellm_params.get("metadata") or {} - - assert "foo" in metadata, "metadata['foo'] should be accessible in callback" - assert metadata["foo"] == "bar" - assert metadata.get("trace_id") == "test-123" - finally: - litellm.callbacks = original_callbacks @pytest.mark.asyncio diff --git a/tests/test_litellm/responses/test_responses_api_bridge_flag.py b/tests/test_litellm/responses/test_responses_api_bridge_flag.py index 2f64cc8debc..642495fab86 100644 --- a/tests/test_litellm/responses/test_responses_api_bridge_flag.py +++ b/tests/test_litellm/responses/test_responses_api_bridge_flag.py @@ -66,6 +66,30 @@ class TestUseResponsesApiBridgeFlag: mock_bridge_handler.assert_called_once() + @patch.object( + import_module("litellm.responses.main").litellm_completion_transformation_handler, "response_api_handler" + ) + @patch.object( + import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config" + ) + def test_provider_affinity_header_is_forwarded_through_bridge(self, mock_get_config, mock_bridge_handler): + mock_get_config.return_value = litellm.OpenAIResponsesAPIConfig() + mock_bridge_handler.return_value = MagicMock() + + litellm.responses( + model="openai/my-custom-model", + input="Hello", + use_chat_completions_api=True, + litellm_session_id="session-bridge", + provider_affinity_header="X-Conversation-Id", + extra_headers={"X-Customer-Header": "customer-value"}, + litellm_logging_obj=MagicMock(), + ) + + forwarded_headers = mock_bridge_handler.call_args.kwargs["extra_headers"] + assert forwarded_headers["X-Conversation-Id"] == "session-bridge" + assert forwarded_headers["X-Customer-Header"] == "customer-value" + @patch.object( import_module("litellm.responses.main").litellm_completion_transformation_handler, "response_api_handler" ) @@ -262,6 +286,126 @@ class TestUseResponsesApiBridgeFlag: assert request_body["messages"] == [{"role": "user", "content": "Hello"}] assert response.output[0].content[0].text == "Answer" + def test_bridge_drops_client_metadata_even_when_allowed_openai_params_names_it( + self, respx_mock: respx.MockRouter + ): + upstream: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "my-custom-model", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + response: Final = litellm.responses( + model="openai/my-custom-model", + input="Hello", + use_chat_completions_api=True, + allowed_openai_params=["client_metadata"], + client_metadata={"turn_id": "turn-1", "thread_id": "thread-1"}, + api_key="fake-provider-api-key", + num_retries=0, + ) + + assert upstream.call_count == 1 + request_body: Final = json.loads(upstream.calls[0].request.read()) + assert "client_metadata" not in request_body + assert request_body["messages"] == [{"role": "user", "content": "Hello"}] + assert response.output[0].content[0].text == "Answer" + + def test_bridge_merges_instructions_and_developer_input_for_databricks(self, respx_mock: respx.MockRouter): + upstream: Final = respx_mock.post("https://example.databricks.test/serving-endpoints/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "my-custom-model", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + response: Final = litellm.responses( + model="databricks/my-custom-model", + instructions="You are terse.", + input=[ + {"role": "developer", "content": [{"type": "input_text", "text": "Skills: none."}]}, + {"role": "user", "content": [{"type": "input_text", "text": "Hello"}]}, + ], + client_metadata={"turn_id": "turn-1", "thread_id": "thread-1"}, + chat_template_kwargs={"thinking": True}, + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + ) + + assert upstream.call_count == 1 + request_body: Final = json.loads(upstream.calls[0].request.read()) + assert request_body["messages"] == [ + { + "role": "system", + "content": [{"type": "text", "text": "You are terse."}, {"type": "text", "text": "Skills: none."}], + }, + {"role": "user", "content": [{"type": "text", "text": "Hello"}]}, + ] + assert "client_metadata" not in request_body + assert request_body["chat_template_kwargs"] == {"thinking": True} + assert response.output[0].content[0].text == "Answer" + + def test_bridge_drops_client_metadata_for_provider_without_native_config(self, respx_mock: respx.MockRouter): + upstream: Final = respx_mock.post("https://example.databricks.test/serving-endpoints/chat/completions").mock( + return_value=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": "my-custom-model", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 1, "total_tokens": 10}, + }, + ) + ) + + response: Final = litellm.responses( + model="databricks/my-custom-model", + input="Hello", + client_metadata={ + "turn_id": "turn-1", + "thread_id": "thread-1", + "session_id": "session-1", + "root_turn_id": "turn-1", + "x-codex-installation-id": "install-1", + "x-codex-turn-metadata": '{"turn_id":"turn-1"}', + }, + chat_template_kwargs={"thinking": True}, + api_base="https://example.databricks.test/serving-endpoints", + api_key="fake-databricks-api-key", + num_retries=0, + ) + + assert upstream.call_count == 1 + request_body: Final = json.loads(upstream.calls[0].request.read()) + assert "client_metadata" not in request_body + assert request_body["chat_template_kwargs"] == {"thinking": True} + assert request_body["messages"] == [{"role": "user", "content": "Hello"}] + assert response.output[0].content[0].text == "Answer" + def test_bridge_keeps_deployment_credentials_while_dropping_unknown_params(self, respx_mock: respx.MockRouter): upstream: Final = respx_mock.post( "https://example-resource.openai.azure.com/openai/deployments/my-deployment/chat/completions", diff --git a/tests/test_litellm/router_strategy/complexity_router/test_context_compaction.py b/tests/test_litellm/router_strategy/complexity_router/test_context_compaction.py new file mode 100644 index 00000000000..c6242f775c8 --- /dev/null +++ b/tests/test_litellm/router_strategy/complexity_router/test_context_compaction.py @@ -0,0 +1,503 @@ +import asyncio +import json +from collections.abc import Iterator, Mapping +from copy import deepcopy +from functools import partial +from typing import Final, Literal + +import httpx +import pytest +import respx + +import litellm +from litellm.llms import compaction as native +from litellm.router_strategy.complexity_router.config import ContextCompactionConfig +from litellm.router_strategy.complexity_router.context_compaction import ( + CompactionState, + Surface, + arm_compaction, + compact_to_fit, + compaction_executor, +) +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES +from litellm.types.router import Deployment + +pytestmark: Final = [pytest.mark.asyncio, pytest.mark.usefixtures("local_model_cost_map")] +SCHEMA: Final = {"type": "object", "properties": {"code": {"type": "string"}}} + + +@pytest.fixture(autouse=True) +def native_catalog(monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None) -> None: + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.setenv("LITELLM_LICENSE", "") + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", False) + monkeypatch.setitem(litellm.model_cost, "summary-fixture", { + "litellm_provider": "anthropic", "mode": "chat", "max_input_tokens": 32000, + "max_output_tokens": 4096, "supports_anthropic_compaction": True, + }) + + +def make_router( + window: int | None = 512, settings: Mapping[str, object] | None = None, *, compactor_window: int = 32000, + conflict: bool = False, output: int | None = 64, + answer_defaults: Mapping[str, object] | None = None, + context_fallback: bool = False, +) -> litellm.Router: + config: Final = { + "tiers": {"SIMPLE": "small", "MEDIUM": "large", "COMPLEX": "large", "REASONING": "large"}, + "keyword_tier_rules": [{"keywords": ["answer", "tail result"], "tier": "SIMPLE"}], + "enable_context_window_escalation": False, "max_tokens_from_tier_model": False, + **(settings or {}), + } + return litellm.Router(model_list=[ + {"model_name": "auto", "litellm_params": { + "model": "auto_router/complexity_router", "complexity_router_config": config, + }}, + {"model_name": "small", "litellm_params": { + "model": "openai/arbitrary-answer", "api_base": "https://answer.test/v1", "api_key": "answer-test", "max_retries": 0, + **(answer_defaults or {}), + }, "model_info": {"id": "pinned-answer", "max_input_tokens": window, "max_output_tokens": output}}, + {"model_name": "large", "litellm_params": { + "model": "anthropic/summary-fixture", "api_base": "https://compact.test", "api_key": "compact-test", + **({"stop": ["deployment policy"]} if conflict else {}), + }, "model_info": {"id": "native-compactor", "max_input_tokens": compactor_window, "max_output_tokens": 4096}}, + {"model_name": "backup", "litellm_params": { + "model": "anthropic/summary-fixture", "api_base": "https://compact.test", "api_key": "backup-test", + }, "model_info": {"id": "backup-compactor", "max_input_tokens": 32000, "max_output_tokens": 4096}}, + ], enable_pre_call_checks=True, num_retries=0, disable_cooldowns=True, + retry_policy={"InternalServerErrorRetries": 1}, + context_window_fallbacks=[{"auto": ["large"]}] if context_fallback else []) + + +def exchange(surface: Surface, phase: str) -> list[dict[str, object]]: + identifier: Final = f"{phase}-call" + result: Final = f"{phase} result" + if surface == "responses": + return [ + {"type": "function_call", "call_id": identifier, "name": "lookup", "arguments": "{}"}, + {"type": "function_call_output", "call_id": identifier, "output": result}, + ] + if surface == "messages": + return [ + {"role": "assistant", "content": [{"type": "tool_use", "id": identifier, "name": "lookup", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": identifier, "content": result}]}, + ] + return [ + {"role": "assistant", "tool_calls": [ + {"id": identifier, "type": "function", "function": {"name": "lookup", "arguments": "{}"}}, + ]}, + {"role": "tool", "tool_call_id": identifier, "content": result}, + ] + + +def history(surface: Surface) -> dict[str, object]: + conversation: Final = [ + {"role": "user", "content": "Project code MAPLE-47. Background detail. " * 150}, + {"role": "assistant", "content": "Recorded"}, + *exchange(surface, "prefix"), + {"role": "user", "content": "Answer with the project code"}, + *exchange(surface, "tail"), + ] + function: Final = {"name": "lookup", "parameters": SCHEMA} + if surface == "responses": + return {"instructions": "Keep the code exact", "tools": [{"type": "function", **function}], "input": [ + {"role": "developer", "content": "Retain the original spelling"}, *conversation, + ]} + if surface == "messages": + return {"system": "Keep the code exact", "tools": [{"name": "lookup", "input_schema": SCHEMA}], "messages": [ + *conversation, + ]} + return {"tools": [{"type": "function", "function": function}], "messages": [ + {"role": "system", "content": "Keep the code exact"}, *conversation, + ]} + + +def native_reply(summary: str = "Project code MAPLE-47", signed: bool = True, truncated: bool = False) -> httpx.Response: + return httpx.Response(200, json={ + "id": "msg_compact", "type": "message", "role": "assistant", "model": "summary-fixture", + "content": [{"type": "compaction", "content": summary, **({"signature": "native-signature"} if signed else {})}], + "stop_reason": "max_tokens" if truncated else "compaction", "usage": {"input_tokens": 0, "output_tokens": 0, "iterations": [ + {"type": "compaction", "input_tokens": 1200, "output_tokens": 20}, + ]}, + }) + + +def answer_reply(request: httpx.Request, expected_model: str = "arbitrary-answer") -> httpx.Response: + payload: Final = json.loads(request.content) + assert payload["model"] == expected_model + assert request.headers["authorization"] == "Bearer answer-test" + if request.url.path.endswith("responses"): + return httpx.Response(200, json={ + "id": "resp_answer", "object": "response", "created_at": 0, "status": "completed", + "model": payload["model"], "output": [{"id": "msg_answer", "type": "message", "role": "assistant", + "status": "completed", "content": [{"type": "output_text", "text": "MAPLE-47", "annotations": []}]}], + "usage": {"input_tokens": 60, "output_tokens": 8, "total_tokens": 68}, + }) + return httpx.Response(200, json={ + "id": "answer", "object": "chat.completion", "created": 0, "model": payload["model"], + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "MAPLE-47"}}], + "usage": {"prompt_tokens": 60, "completion_tokens": 8, "total_tokens": 68}, + }) + + +@pytest.fixture +def wire() -> Iterator[tuple[respx.Route, respx.Route]]: + with respx.mock(assert_all_called=False) as transport: + compactor: Final = transport.post("https://compact.test/v1/messages").mock(return_value=native_reply()) + answer: Final = transport.route(method="POST", host="answer.test").mock(side_effect=answer_reply) + yield compactor, answer + + +async def invoke(router: litellm.Router, surface: Surface, payload: Mapping[str, object], retries: int = 0) -> object: + if surface == "responses": + return await router.aresponses(model="auto", max_output_tokens=64, num_retries=retries, **payload) + if surface == "messages": + return await router.aanthropic_messages(model="auto", max_tokens=64, num_retries=retries, **payload) + return await router.acompletion(model="auto", max_tokens=64, num_retries=retries, **payload) + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +@pytest.mark.parametrize("near", [False, True]) +@pytest.mark.parametrize("configured", [False, True]) +async def test_all_surfaces_compact_and_keep_selected_answerer( + wire: tuple[respx.Route, respx.Route], surface: Surface, near: bool, configured: bool, +) -> None: + payload: Final = history(surface) + original: Final = deepcopy(payload) + counted: Final = make_router()._count_pre_call_check_tokens(payload.get("messages"), payload.get("input"), payload) + window: Final = int((counted + 32) / ContextCompactionConfig().trigger_ratio) + 1 if near else 512 + assert (counted < window) is near + settings: Final = {"enable_context_window_escalation": True, + **({"context_compaction": {"model": "large", "max_tokens": 512}} if configured else {})} + router: Final = make_router(window, settings) + captured: Final = asyncio.Queue[Mapping[str, object]]() + compactor, answer = wire + retry: Final = near and configured + + def answer_after_retry(request: httpx.Request) -> httpx.Response: + return httpx.Response(500, json={"error": {"message": "retry answer"}}) if answer.call_count == 0 else answer_reply(request) + + if retry: + answer.mock(side_effect=answer_after_retry) + + async def execute( + protocol: native.CompactionProtocol, request: Mapping[str, object], parent_model: str | None = None + ) -> Mapping[str, object]: + assert parent_model == "auto" + result: Final = await native.dispatch(router, protocol, request) + captured.put_nowait(result) + return result + + token: Final = compaction_executor.set(execute) + try: + response: Final = await invoke(router, surface, payload, retries=int(retry)) + finally: + compaction_executor.reset(token) + assert compactor.call_count == captured.qsize() == 1 + assert answer.call_count == router.total_calls["openai/arbitrary-answer"] == 1 + int(retry) + compact_request: Final = compactor.calls[0].request + compact_body: Final = json.loads(compact_request.content) + answer_body: Final = json.loads(answer.calls[0].request.content) + assert answer_body == json.loads(answer.calls[-1].request.content) + assert compact_body["model"] == "summary-fixture" and compact_body["compaction"] == {"type": "summarize"} + assert ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value in compact_request.headers["anthropic-beta"].split(",") + assert compact_body["max_tokens"] == (512 if configured else ContextCompactionConfig().max_tokens) + assert "Background detail" in str(compact_body) and "tail-call" not in str(compact_body) + assert "prefix-call" in str(compact_body) and "prefix result" in str(compact_body) + assert "Keep the code exact" in str(compact_body["system"]) + assert compact_body["tools"][0]["input_schema"] == SCHEMA + assert "MAPLE-47" in str(answer_body) and "Background detail" not in str(answer_body) + assert "prefix-call" not in str(answer_body) + assert "native-signature" not in str(answer_body) and "compaction" not in answer_body + assert "tail-call" in str(answer_body) and "tail result" in str(answer_body) + assert "MAPLE-47" in str(response) + usage: Final = captured.get_nowait()["usage"] + if surface == "messages": + assert usage["iterations"][0]["input_tokens"] == 1200 and usage["iterations"][0]["output_tokens"] == 20 + else: + assert usage["prompt_tokens"] == 1200 and usage["completion_tokens"] == 20 + if surface == "responses": + assert answer_body["input"][-3:] == original["input"][-3:] + assert answer_body["input"][0] == original["input"][0] + assert answer_body["instructions"] == original["instructions"] and answer_body["tools"] == original["tools"] + assert payload == original + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +@pytest.mark.parametrize("mode", ["fitting", "false", "null"]) +async def test_fitting_and_disabled_requests_do_not_compact( + wire: tuple[respx.Route, respx.Route], surface: Surface, mode: str, +) -> None: + settings: Final = {} if mode == "fitting" else {"context_compaction": False if mode == "false" else None} + payload: Final = history(surface) + original: Final = deepcopy(payload) + counted: Final = make_router()._count_pre_call_check_tokens(payload.get("messages"), payload.get("input"), payload) + await invoke(make_router(20000 if mode == "fitting" else counted + 64, settings), surface, payload) + compactor, answer = wire + assert compactor.call_count == 0 and answer.call_count == 1 + assert "Background detail" in answer.calls[0].request.content.decode() + assert payload == original + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +@pytest.mark.parametrize("reason", ["single", "unclosed", "no_compactor"]) +async def test_fitting_request_survives_unavailable_compaction( + wire: tuple[respx.Route, respx.Route], surface: Surface, reason: str, +) -> None: + key: Final = "input" if surface == "responses" else "messages" + items: Final = [{"role": "user", "content": "Answer with MAPLE-47. Detail. " * 80}] + payload: Final = history(surface) if reason == "no_compactor" else {key: ( + items if reason == "single" else [*items, *exchange(surface, "open")[:1], {"role": "user", "content": "Answer"}] + )} + counted: Final = make_router()._count_pre_call_check_tokens(payload.get("messages"), payload.get("input"), payload) + settings: Final = {"tiers": {"SIMPLE": "small", "MEDIUM": "small", "COMPLEX": "small", "REASONING": "small"}} if reason == "no_compactor" else {} + await invoke(make_router(counted + 1, settings), surface, payload) + compactor, answer = wire + assert compactor.call_count == 0 and answer.call_count == 1 + assert "Detail" in str(answer.calls[0].request.content) or "Background detail" in str(answer.calls[0].request.content) + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +async def test_uncompactable_overflow_uses_explicit_context_fallback( + wire: tuple[respx.Route, respx.Route], surface: Surface, +) -> None: + compactor, answer = wire + compactor.mock(return_value=httpx.Response(200, json={**native_reply().json(), + "content": [{"type": "text", "text": "MAPLE-47"}], "stop_reason": "end_turn"})) + payload: Final = {"input" if surface == "responses" else "messages": [ + {"role": "user", "content": "Answer with MAPLE-47. Detail. " * 150}, + ]} + await invoke(make_router(context_fallback=True), surface, payload) + assert answer.call_count == 0 and compactor.call_count == 1 + assert "compaction" not in json.loads(compactor.calls[0].request.content) + + +@pytest.mark.parametrize("escalate", [False, True]) +async def test_no_native_compactor_respects_explicit_escalation( + wire: tuple[respx.Route, respx.Route], monkeypatch: pytest.MonkeyPatch, escalate: bool, +) -> None: + monkeypatch.setitem(litellm.model_cost["summary-fixture"], "supports_anthropic_compaction", False) + router: Final = make_router(settings={"enable_context_window_escalation": escalate}) + compactor, answer = wire + compactor.mock(return_value=httpx.Response(200, json={**native_reply().json(), + "content": [{"type": "text", "text": "MAPLE-47"}], "stop_reason": "end_turn"})) + if not escalate: + with pytest.raises(litellm.ContextWindowExceededError, match="No configured compactor"): + await invoke(router, "chat", history("chat")) + assert compactor.call_count == 0 + else: + await invoke(router, "chat", history("chat")) + assert compactor.call_count == 1 + assert "compaction" not in json.loads(compactor.calls[0].request.content) + assert answer.call_count == 0 + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +async def test_undersized_native_compactor_does_not_block_explicit_escalation( + wire: tuple[respx.Route, respx.Route], surface: Surface, +) -> None: + router: Final = make_router(compactor_window=512, settings={ + "enable_context_window_escalation": True, + "tiers": {"SIMPLE": "small", "MEDIUM": "large", "COMPLEX": "wide", "REASONING": "wide"}, + }) + router.add_deployment(Deployment( + model_name="wide", litellm_params={ + "model": "openai/wide-answer", "api_base": "https://answer.test/v1", "api_key": "answer-test", + }, model_info={"id": "wide-answer", "max_input_tokens": 32000, "max_output_tokens": 64}, + )) + compactor, answer = wire + answer.mock(side_effect=partial(answer_reply, expected_model="wide-answer")) + await invoke(router, surface, history(surface)) + assert compactor.call_count == 0 and answer.call_count == 1 + assert json.loads(answer.calls[0].request.content)["model"] == "wide-answer" + + +@pytest.mark.parametrize( + ("surface", "failure"), + [(surface, failure) for surface in ("chat", "messages", "responses") for failure in ("unsigned", "oversized", "provider")] + + [("messages", "truncated")], +) +async def test_bad_native_result_never_reaches_answerer( + wire: tuple[respx.Route, respx.Route], surface: Surface, failure: str, +) -> None: + compactor, answer = wire + reply: Final = httpx.Response(500, json={"error": {"type": "api_error", "message": "failed"}}) if failure == "provider" else native_reply( + "too large " * 2000 if failure == "oversized" else "MAPLE-47", signed=failure != "unsigned", + truncated=failure == "truncated", + ) + compactor.mock(return_value=reply) + with pytest.raises((litellm.BadRequestError, litellm.InternalServerError)): + await invoke(make_router(), surface, history(surface)) + assert compactor.call_count == 1 and answer.call_count == 0 + + +@pytest.mark.parametrize("failure", ["item", "content", "tools", "unclosed", "missing", "duplicate", "instructions", "retained"]) +@pytest.mark.parametrize("needed", [False, True]) +async def test_unsafe_responses_reject_only_when_compaction_needed( + wire: tuple[respx.Route, respx.Route], failure: str, needed: bool, +) -> None: + payload: Final = history("responses") + extra: Final = { + "item": [{"type": "computer_call", "call_id": "opaque-tool"}], + "content": [{"role": "assistant", "content": [{"type": "refusal", "refusal": "cannot"}]}], + "unclosed": [{"type": "function_call", "call_id": "unclosed", "name": "lookup", "arguments": "{}"}], + "missing": [{"type": "function_call", "name": "lookup", "arguments": "{}"}], + "duplicate": exchange("responses", "prefix"), + "instructions": [{"role": "developer", "content": "Changed instructions"}], + } + request: Final = { + **payload, "input": [*payload["input"][:3], *extra.get(failure, []), *payload["input"][3:]], + **({"tools": [{"type": "computer_use_preview", "display_width": 800, "display_height": 600}]} if failure == "tools" else {}), + **({"instructions": "Keep every instruction " * 600} if failure == "retained" else {}), + } + if needed: + with pytest.raises(litellm.BadRequestError, match="Context compaction"): + await invoke(make_router(), "responses", request) + assert all(route.call_count == 0 for route in wire) + else: + await invoke(make_router(20000), "responses", request) + compactor, answer = wire + assert compactor.call_count == 0 and answer.call_count == 1 + + +@pytest.mark.parametrize("owned", [ + {"previous_response_id": "resp_parent"}, {"conversation": "conv_parent"}, + {"context_management": [{"type": "compaction", "compact_threshold": 1000}]}, {"compaction": {"type": "summarize"}}, + {"input": [{"type": "reasoning", "encrypted_content": "opaque"}]}, + {"input": [{"type": "reasoning", "summary": [{"type": "summary_text", "text": "prior reasoning"}]}]}, + {"input": [{"type": "compaction", "encrypted_content": "opaque"}]}, + {"input": [{"type": "item_reference", "id": "item_parent"}]}, + {"input": [{"role": "assistant", "content": "visible", "encrypted_content": "opaque"}]}, + {"input": [{"role": "user", "content": [{"type": "encrypted_content", "encrypted_content": "opaque"}]}]}, +]) +@pytest.mark.parametrize("arm_first", [False, True]) +async def test_client_owned_history_bypasses_compaction( + wire: tuple[respx.Route, respx.Route], owned: Mapping[str, object], arm_first: bool, +) -> None: + router: Final = make_router() + deployment: Final = router.get_deployment(model_id="pinned-answer") + assert deployment is not None + state: Final = CompactionState() + request: Final = {**history("responses"), **owned, "model": "small", "max_tokens": 64, "_context_compaction_state": state} + original: Final = deepcopy({key: value for key, value in request.items() if key != "_context_compaction_state"}) + before_defaults: Final = {"_context_compaction_state": state} if arm_first else request + await arm_compaction(before_defaults, ContextCompactionConfig(), ("large",)) + counted: Final = router._count_pre_call_check_tokens(None, request["input"], request) + if arm_first and counted > 512: + with pytest.raises(litellm.ContextWindowExceededError): + await compact_to_fit(router, deployment.model_dump(), request, "responses") + else: + result: Final = await compact_to_fit(router, deployment.model_dump(), request, "responses") + assert result is request + assert {key: value for key, value in request.items() if key != "_context_compaction_state"} == original + assert all(route.call_count == 0 for route in wire) + + +@pytest.mark.parametrize("surface", ["chat", "messages", "responses"]) +@pytest.mark.parametrize("source", ["request", "deployment"]) +async def test_client_managed_overflow_keeps_context_window_admission( + wire: tuple[respx.Route, respx.Route], surface: Surface, source: str, +) -> None: + managed: Final = {"context_management": {"edits": []}} + router: Final = make_router(answer_defaults=managed if source == "deployment" else None) + payload: Final = {**history(surface), **(managed if source == "request" else {})} + compactor, answer = wire + if source == "deployment": + with pytest.raises(litellm.ContextWindowExceededError): + await invoke(router, surface, payload) + assert compactor.call_count == 0 + else: + await invoke(router, surface, payload) + assert compactor.call_count == 1 + assert "compaction" not in json.loads(compactor.calls[0].request.content) + assert answer.call_count == 0 + + +@pytest.mark.parametrize("case", ["conflicting_defaults", "small_window", "capability_false", "capability_missing"]) +async def test_automatic_compactor_skips_conflicts_and_requires_capacity_and_capability( + wire: tuple[respx.Route, respx.Route], monkeypatch: pytest.MonkeyPatch, case: str, +) -> None: + if case.startswith("capability"): + metadata: Final = {key: value for key, value in litellm.model_cost["summary-fixture"].items() + if key != "supports_anthropic_compaction"} + monkeypatch.setitem(litellm.model_cost, "summary-fixture", { + **metadata, **({"supports_anthropic_compaction": False} if case == "capability_false" else {}), + }) + conflict: Final = case == "conflicting_defaults" + settings: Final = {"tiers": {"SIMPLE": "small", "MEDIUM": "large", "COMPLEX": "backup", "REASONING": "backup"}} if conflict else {} + router: Final = make_router(settings=settings, conflict=conflict, compactor_window=512 if case == "small_window" else 32000) + if conflict: + await invoke(router, "chat", history("chat")) + compactor, answer = wire + assert compactor.call_count == answer.call_count == 1 + assert compactor.calls[0].request.headers["x-api-key"] == "backup-test" + assert "stop_sequences" not in json.loads(compactor.calls[0].request.content) + else: + with pytest.raises(litellm.BadRequestError, match="No configured compactor"): + await invoke(router, "chat", history("chat")) + assert all(route.call_count == 0 for route in wire) + + +@pytest.mark.parametrize("unknown_output", [False, True]) +@pytest.mark.parametrize("overflow", [False, True]) +async def test_unusable_output_budget_still_enforces_known_input_window( + wire: tuple[respx.Route, respx.Route], unknown_output: bool, overflow: bool, +) -> None: + payload: Final = history("chat") + counted: Final = make_router(output=None)._count_pre_call_check_tokens(payload["messages"], None, payload) + window: Final = 512 if overflow else counted + 64 + output: Final = None if unknown_output else window + router: Final = make_router(window, output=output) + deployment: Final = router.get_deployment(model_id="pinned-answer") + assert deployment is not None + state: Final = CompactionState(config=ContextCompactionConfig(), candidates=("large",)) + request: Final = {**payload, "model": "small", "max_tokens": output, "_context_compaction_state": state} + if overflow: + with pytest.raises(litellm.BadRequestError, match="known input window"): + await compact_to_fit(router, deployment.model_dump(), request, "chat") + else: + assert await compact_to_fit(router, deployment.model_dump(), request, "chat") is request + assert all(route.call_count == 0 for route in wire) + + +@pytest.mark.parametrize("outcome", ["success", "timeout", "cancel"]) +async def test_retry_reuses_summary_or_terminal_cancellation(outcome: Literal["success", "timeout", "cancel"]) -> None: + router: Final = make_router() + deployment: Final = router.get_deployment(model_id="pinned-answer") + assert deployment is not None + state: Final = CompactionState(config=ContextCompactionConfig(model="large", max_tokens=512, timeout_seconds=0.02)) + request: Final = {**history("messages"), "model": "small", "max_tokens": 64, "_context_compaction_state": state} + calls: Final = asyncio.Queue[None]() + started: Final = asyncio.Event() + stopped: Final = asyncio.Event() + + async def execute( + protocol: native.CompactionProtocol, payload: Mapping[str, object], parent_model: str | None = None + ) -> Mapping[str, object]: + calls.put_nowait(None) + started.set() + try: + return native_reply().json() if outcome == "success" else await asyncio.Future[Mapping[str, object]]() + finally: + stopped.set() + + token: Final = compaction_executor.set(execute) + try: + first: Final = asyncio.create_task(compact_to_fit(router, deployment.model_dump(), request, "messages")) + await asyncio.wait_for(started.wait(), timeout=2) + if outcome == "cancel": + first.cancel() + if outcome == "success": + assert await first == await compact_to_fit(router, deployment.model_dump(), request, "messages") + changed: Final = {**request, "messages": [{"role": "user", "content": "new history"}, *request["messages"]]} + with pytest.raises(litellm.BadRequestError, match="History changed"): + await compact_to_fit(router, deployment.model_dump(), changed, "messages") + else: + error: Final = asyncio.CancelledError if outcome == "cancel" else asyncio.TimeoutError + with pytest.raises(error): + await first + with pytest.raises(error): + await compact_to_fit(router, deployment.model_dump(), request, "messages") + assert calls.qsize() == 1 and stopped.is_set() + finally: + compaction_executor.reset(token) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 97e7a799053..4e146b59b61 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -79,6 +79,7 @@ from litellm.router_strategy.complexity_router.jev_classifier import ( JevSystemOneResponse, JevUsage, ) +from litellm.router_strategy.complexity_router.llm_v2 import LLM_V2_PROMPT_VERSION from litellm.router_strategy.complexity_router.tier_predictor import ( TierGlobalStatistic, TrainedTierArtifact, @@ -3240,6 +3241,41 @@ class TestCapabilityClassifier: ) assert response.model == "capable-model" assert response.routing_decision["cause"] == "capability_classifier_fallback" + assert "classifier_p_solve" not in response.routing_decision + assert "classifier_threshold" not in response.routing_decision + + @pytest.mark.asyncio + @pytest.mark.parametrize("bypass", ("literal_keyword_match", "session_affinity_pin", "housekeeping")) + async def test_bypasses_do_not_reuse_the_previous_capability_forecast( + self, + mock_router_instance: MagicMock, + bypass: Literal["literal_keyword_match", "session_affinity_pin", "housekeeping"], + ) -> None: + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response(_capability_reply(p_solve=0.8))) + mock_router_instance.cache = DualCache() + router: Final = self._router( + mock_router_instance, + session_affinity=bypass == "session_affinity_pin", + keyword_tier_rules=[{"keywords": ["quick lookup"], "tier": "SIMPLE"}], + ) + original: Final = await router.async_pre_routing_hook( + model="capability-router", + request_kwargs={"metadata": {"session_id": "forecast-bypass"}}, + messages=[{"role": "user", "content": "Hello!"}], + ) + result: Final = await router.async_pre_routing_hook( + model="capability-router", + request_kwargs={"metadata": {"session_id": "forecast-bypass"}}, + messages=[{"role": "user", "content": TITLE_ASK if bypass == "housekeeping" else "quick lookup"}], + ) + + assert original is not None and original.routing_decision is not None + assert original.routing_decision["classifier_p_solve"] == 0.8 + assert result is not None and result.routing_decision is not None + assert result.routing_decision["cause"] == bypass + assert "classifier_p_solve" not in result.routing_decision + assert "classifier_threshold" not in result.routing_decision + mock_router_instance.acompletion.assert_awaited_once() CUSTOM_TIER_LABELS: Dict[str, str] = { @@ -14686,6 +14722,121 @@ class TestModalityRouting: @pytest.mark.usefixtures("local_model_cost_map") class TestHealthFallbackDispatch: + @pytest.mark.asyncio + @pytest.mark.parametrize("classifier", ("capability", "llm_v2")) + @pytest.mark.parametrize("calibrated", (False, True), ids=("raw", "calibrated")) + @pytest.mark.parametrize("rewrite", ("modality_escalation", "health_failover", "health_default_fallback")) + async def test_classifier_forecasts_survive_placement_rewrites( + self, + classifier: Literal["capability", "llm_v2"], + calibrated: bool, + rewrite: Literal["modality_escalation", "health_failover", "health_default_fallback"], + ) -> None: + calibration: Final = {"slope": 0.8, "intercept": 0.1} + classifier_config: Final = ( + { + "capability_classifier_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "base_threshold": 0.0, + "threshold_step": 0.1, + **({"calibration": {"version": "test-v1", **calibration}} if calibrated else {}), + } + } + if classifier == "capability" + else { + "llm_v2_config": { + "efficient_profile": "Small coding solver", + "capable_profile": "Large coding solver", + "harness": "Repository tools", + "max_quality_gap": 0.0, + **( + { + "calibration": { + "version": "test-v1", + "prompt_version": LLM_V2_PROMPT_VERSION, + "efficient": calibration, + "capable": calibration, + } + } + if calibrated + else {} + ), + } + } + ) + router: Final = self._router( + config={ + "classifier_type": classifier, + "classifier_llm_config": {"model": "fallback", "timeout_ms": 10000}, + "tiers": {"SIMPLE": "primary", "REASONING": "peer"}, + "tier_labels": {"SIMPLE": "Entry", "REASONING": "Advanced"}, + "modality_routing": True, + **classifier_config, + } + ) + verdict: Final = ( + _capability_reply(p_solve=0.0) + if classifier == "capability" + else json.dumps( + { + "crux": "Preserve existing behavior", + "demands": {"reasoning": "routine", "scope": "localized", "specification": "clear"}, + "verification": "relevant", + "forecasts": { + "efficient": {"likely_failure": "Miss an edge case", "p_solve": 0.0}, + "capable": {"likely_failure": "Miss an edge case", "p_solve": 0.0}, + }, + } + ) + ) + judge_response: Final = litellm.ModelResponse( + choices=[{"message": {"role": "assistant", "content": verdict}}], + usage={"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11}, + ) + with respx.mock(assert_all_mocked=True) as upstream: + upstream.post(host="fallback.test").respond(json=judge_response.model_dump()) + original: Final = await router.async_pre_routing_hook( + model="health-router", request_kwargs={}, messages=[{"role": "user", "content": "Hello!"}] + ) + for deployment in router.model_list: + deployment["model_info"]["supports_vision"] = ( + rewrite != "modality_escalation" or deployment["model_name"] != "primary" + ) + if rewrite != "modality_escalation": + self._unavailable(router, "primary-id", "cooldown") + if rewrite == "health_default_fallback": + self._unavailable(router, "peer-id", "cooldown") + result: Final = await router.async_pre_routing_hook( + model="health-router", request_kwargs={}, messages=TestModalityRouting.IMAGE_MESSAGE + ) + + assert original is not None and original.routing_decision is not None + assert original.model == "primary" + assert result is not None and result.routing_decision is not None + decision: Final = result.routing_decision + assert decision["cause"] == rewrite + assert result.model == ("fallback" if rewrite == "health_default_fallback" else "peer") + expected: Final = { + field: value for field, value in original.routing_decision.items() if field.startswith("classifier_") + } + assert expected["classifier_p_solve" if classifier == "capability" else "classifier_efficient_p_solve"] == 0.0 + assert ("classifier_calibration_version" in expected) is calibrated + assert {field: value for field, value in decision.items() if field.startswith("classifier_")} == expected + if rewrite == "health_default_fallback": + assert "tier" not in decision and "tier_label" not in decision + else: + assert decision["tier"] == "REASONING" + assert decision["tier_label"] == "Advanced" + redacted: Final = Router._redact_prompt_text_if_needed( + request_kwargs={"metadata": {"headers": {"x-litellm-enable-message-redaction": True}}}, + routing_decision=decision, + ) + assert "classifier_crux" not in redacted and "signals" not in redacted + assert {field: value for field, value in redacted.items() if field.startswith("classifier_")} == { + field: value for field, value in expected.items() if field != "classifier_crux" + } + @pytest.mark.asyncio @pytest.mark.parametrize("peer", (True, False), ids=("peer_failover", "default_fallback")) async def test_health_rewrites_preserve_the_original_heuristic_v2_forecast(self, peer: bool) -> None: @@ -15166,6 +15317,7 @@ class TestHealthFallbackDispatch: router: Final = self._router( config={ + "context_compaction": False, "tiers": {"SIMPLE": "primary", "MEDIUM": "peer", "COMPLEX": "large"}, "enable_context_window_escalation": True, } @@ -15248,6 +15400,7 @@ class TestHealthFallbackDispatch: async def test_modality_default_must_also_fit_context(self, default_fits: bool) -> None: router: Final = self._router( config={ + "context_compaction": False, "modality_routing": True, "tiers": {"SIMPLE": "primary"}, "enable_context_window_escalation": True, diff --git a/tests/test_litellm/router_strategy/test_llm_v2.py b/tests/test_litellm/router_strategy/test_llm_v2.py index 27d31cbe640..6fb6df3265d 100644 --- a/tests/test_litellm/router_strategy/test_llm_v2.py +++ b/tests/test_litellm/router_strategy/test_llm_v2.py @@ -477,6 +477,10 @@ async def test_user_turn_mode_reuses_forecast_until_a_new_user_requirement() -> assert first.model == second.model == "efficient" assert first.routing_decision["cause"] == "llm_v2_classifier" assert first.routing_decision["classifier_cost"] == 0.001 + assert second is not None and second.routing_decision is not None + assert second.routing_decision["cause"] == "user_turn_continuation" + assert "classifier_efficient_p_solve" not in second.routing_decision + assert "classifier_capable_p_solve" not in second.routing_decision client.acompletion.assert_awaited_once() client.acompletion.return_value = _response(_verdict(0.3, 0.9).model_dump_json()) updated: Final = await router.async_pre_routing_hook( diff --git a/tests/test_litellm/router_strategy/test_router_routing_groups.py b/tests/test_litellm/router_strategy/test_router_routing_groups.py index 506563a82fb..425f68dda18 100644 --- a/tests/test_litellm/router_strategy/test_router_routing_groups.py +++ b/tests/test_litellm/router_strategy/test_router_routing_groups.py @@ -9,7 +9,8 @@ import asyncio import datetime import time import uuid -from collections.abc import Callable +from collections.abc import Callable, Mapping +from typing import Final, Literal from unittest.mock import patch import pytest @@ -18,7 +19,7 @@ from pydantic import ValidationError import litellm from litellm import Router from litellm.integrations.custom_logger import CustomLogger -from litellm.types.router import RoutingGroup, RoutingStrategy +from litellm.types.router import DeploymentTypedDict, FallbackAccessCheck, RoutingGroup, RoutingStrategy from litellm.utils import Rules, function_setup @@ -1676,3 +1677,475 @@ async def test_group_call_429_cools_down_member_across_retries(): ) cooldown_ids = await _call_and_get_cooldowns(router, "quality") assert "deploy-3" in cooldown_ids + + +def _priority_group( + name: str = "priority-group", primary: int = 1, backup: int = 2 +) -> RoutingGroup: + return RoutingGroup.model_validate({ + "group_name": name, + "models": ["filtered-model", "other-model"], + "routing_strategy": "priority", + "model_priorities": {"filtered-model": primary, "other-model": backup}, + }) + + +def _priority_deployments( + primary_response: str = "primary", primary_blocked: bool = False, backup_response: str = "backup" +) -> list[DeploymentTypedDict]: + return [ + { + **deployment, + "litellm_params": { + **deployment["litellm_params"], + "mock_response": ( + primary_response if deployment["model_name"] == "filtered-model" else backup_response + ), + "order": 10 if deployment["model_name"] == "filtered-model" else 1, + }, + "model_info": { + **deployment["model_info"], + "blocked": primary_blocked and deployment["model_name"] == "filtered-model", + }, + } + for deployment in _model_list() + ] + + +def test_priority_group_affinity_scope_follows_group_aliases_and_settings_reload() -> None: + router: Final = Router( + model_list=_priority_deployments(), + routing_groups=[ + _priority_group(), + RoutingGroup(group_name="legacy", models=["filtered-model"], routing_strategy="simple-shuffle"), + ], + model_group_alias={"priority-alias": "priority-group"}, + ) + requested_models: Final = ( + "priority-group", "priority-alias", "filtered-model", "other-model", "legacy", "missing" + ) + assert tuple(router._is_priority_routing_group(model) for model in requested_models) == ( + True, True, False, False, False, False + ) + + router.update_settings(routing_groups=[{ + "group_name": "priority-group", + "models": ["filtered-model", "other-model"], + "routing_strategy": "simple-shuffle", + }]) + assert tuple(router._is_priority_routing_group(model) for model in requested_models) == (False,) * 6 + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("requested_model", ["priority-group", "priority-alias"]) +@pytest.mark.asyncio +async def test_priority_group_always_uses_primary_when_healthy( + asynchronous: bool, requested_model: str +) -> None: + router: Final = Router( + model_list=_priority_deployments(), + routing_groups=[_priority_group()], + model_group_alias={"priority-alias": "priority-group"}, + num_retries=0, + ) + request: Final = {"model": requested_model, "messages": [{"role": "user", "content": "hi"}]} + response: Final = ( + await router.acompletion(**request) if asynchronous else router.completion(**request) + ) + assert response.choices[0].message.content == "primary" + + +@pytest.mark.parametrize("requested_model", ["priority-group", "priority-alias"]) +@pytest.mark.asyncio +async def test_priority_group_fails_over_without_retries_and_leaves_direct_calls_unchanged( + requested_model: str, +) -> None: + router: Final = Router( + model_list=_priority_deployments(primary_response="litellm.RateLimitError"), + routing_groups=[_priority_group()], + model_group_alias={"priority-alias": "priority-group"}, + num_retries=0, + disable_cooldowns=True, + ) + response: Final = await router.acompletion( + model=requested_model, messages=[{"role": "user", "content": "hi"}] + ) + assert response.choices[0].message.content == "backup" + with pytest.raises(litellm.RateLimitError): + await router.acompletion(model="filtered-model", messages=[{"role": "user", "content": "hi"}]) + + +@pytest.mark.asyncio +async def test_opposite_priority_groups_preserve_deployments_and_legacy_member_policy() -> None: + router: Final = Router( + model_list=_priority_deployments(), + routing_strategy="latency-based-routing", + routing_groups=[ + RoutingGroup(group_name="legacy", models=["filtered-model"], routing_strategy="least-busy"), + _priority_group(), + _priority_group("reverse-group", primary=2, backup=1), + ], + ) + before: Final = router.get_model_list(model_name="filtered-model") + forward: Final = await router.acompletion( + model="priority-group", messages=[{"role": "user", "content": "hi"}] + ) + reverse: Final = await router.acompletion( + model="reverse-group", messages=[{"role": "user", "content": "hi"}] + ) + assert (forward.choices[0].message.content, reverse.choices[0].message.content) == ("primary", "backup") + assert router._get_routing_context("filtered-model")[0] == "least-busy" + assert router._get_routing_context("other-model")[0] == "latency-based-routing" + assert router.get_model_list(model_name="filtered-model") == before + assert all(deployment["litellm_params"]["order"] == 10 for deployment in before) + + +@pytest.mark.parametrize("backup_priority, expected", [(1, "backup"), (2, "primary")]) +@pytest.mark.asyncio +async def test_priority_group_weights_select_only_within_the_first_eligible_level( + backup_priority: int, expected: str +) -> None: + router: Final = Router( + model_list=_priority_deployments(), routing_groups=[_priority_group(backup=backup_priority)] + ) + response: Final = await router.acompletion( + model="priority-group", + messages=[{"role": "user", "content": "hi"}], + _router_weights={"priority-group": {"deploy-1": 0, "deploy-2": 0, "deploy-3": 1}}, + ) + assert response.choices[0].message.content == expected + + +@pytest.mark.asyncio +async def test_priority_group_skips_paused_primary_and_returns_to_it_after_recovery() -> None: + router: Final = Router( + model_list=_priority_deployments(primary_blocked=True), routing_groups=[_priority_group()] + ) + paused: Final = await router.acompletion( + model="priority-group", messages=[{"role": "user", "content": "hi"}] + ) + router.set_model_list(_priority_deployments()) + recovered: Final = await router.acompletion( + model="priority-group", messages=[{"role": "user", "content": "hi"}] + ) + assert (paused.choices[0].message.content, recovered.choices[0].message.content) == ("backup", "primary") + + +@pytest.mark.parametrize("controls", [{"disable_fallbacks": True}, {"max_fallbacks": 0}]) +@pytest.mark.asyncio +async def test_priority_group_respects_request_fallback_controls(controls: dict[str, object]) -> None: + router: Final = Router( + model_list=_priority_deployments(primary_response="litellm.RateLimitError"), + routing_groups=[_priority_group()], + num_retries=0, + disable_cooldowns=True, + ) + with pytest.raises(litellm.RateLimitError): + await router.acompletion( + model="priority-group", messages=[{"role": "user", "content": "hi"}], **controls + ) + + +@pytest.mark.parametrize( + "priorities", + [None, {}, {"filtered-model": 1}, {"filtered-model": 1, "other-model": 2, "extra": 3}], +) +def test_priority_group_requires_exact_member_priorities(priorities: object) -> None: + with pytest.raises(ValidationError): + RoutingGroup.model_validate({ + "group_name": "priority-group", + "models": ["filtered-model", "other-model"], + "routing_strategy": "priority", + "model_priorities": priorities, + }) + + +@pytest.mark.parametrize("priority", [True, 0, -1, 1.5, "1", 9007199254740992]) +def test_priority_group_rejects_invalid_priority_values(priority: object) -> None: + with pytest.raises(ValidationError): + RoutingGroup.model_validate({ + "group_name": "priority-group", + "models": ["filtered-model"], + "routing_strategy": "priority", + "model_priorities": {"filtered-model": priority}, + }) + + +@pytest.mark.parametrize("models", [[], ["filtered-model", "filtered-model"]]) +def test_priority_group_requires_nonempty_unique_members(models: list[str]) -> None: + with pytest.raises(ValidationError): + RoutingGroup.model_validate({ + "group_name": "priority-group", + "models": models, + "routing_strategy": "priority", + "model_priorities": {model: 1 for model in models}, + }) + + +@pytest.mark.parametrize( + "changes", [{"routing_strategy": "simple-shuffle"}, {"routing_strategy_args": {"ttl": 60}}] +) +def test_priority_group_rejects_conflicting_strategy_settings(changes: dict[str, object]) -> None: + with pytest.raises(ValidationError): + RoutingGroup.model_validate({ + "group_name": "priority-group", + "models": ["filtered-model"], + "routing_strategy": "priority", + "model_priorities": {"filtered-model": 1}, + **changes, + }) + + +@pytest.mark.parametrize("group_name", ["filtered-model", "priority-alias"]) +def test_priority_group_rejects_names_shadowed_by_a_model_or_alias(group_name: str) -> None: + with pytest.raises(ValueError, match=r"shadow|collid|conflict"): + Router( + model_list=_priority_deployments(), + routing_groups=[_priority_group(name=group_name)], + model_group_alias={"priority-alias": "filtered-model"}, + ) + + +def test_priority_is_rejected_as_a_top_level_strategy() -> None: + with pytest.raises(ValueError, match="routing_strategy"): + _build_router(routing_strategy="priority") + + +@pytest.mark.asyncio +async def test_priority_group_settings_roundtrip_replaces_the_order() -> None: + router: Final = Router(model_list=_priority_deployments(), routing_groups=[_priority_group()]) + replacement: Final = _priority_group(primary=9007199254740991, backup=1) + router.update_settings(routing_groups=[replacement.model_dump()]) + stored: Final = router.get_settings()["routing_groups"] + assert stored == [replacement.model_dump()] + response: Final = await router.acompletion( + model="priority-group", messages=[{"role": "user", "content": "hi"}] + ) + assert response.choices[0].message.content == "backup" + + +def _priority_auto_router_deployment() -> DeploymentTypedDict: + return { + "model_name": "smart-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "classifier_type": "heuristic", + "adaptive": False, + "tiers": { + tier: "priority-group" + for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING") + }, + }, + "complexity_router_default_model": "filtered-model", + }, + } + + +@pytest.mark.asyncio +async def test_auto_router_selected_priority_group_fails_over_inside_the_group() -> None: + router: Final = Router( + model_list=[ + *_priority_deployments(primary_response="litellm.RateLimitError"), + _priority_auto_router_deployment(), + ], + routing_groups=[_priority_group()], + num_retries=0, + disable_cooldowns=True, + ) + response: Final = await router.acompletion( + model="smart-router", messages=[{"role": "user", "content": "hi"}] + ) + assert response.choices[0].message.content == "backup" + + +@pytest.mark.parametrize("endpoint_request", [{"input": "hi"}, {"messages": [{"role": "user", "content": "hi"}]}]) +@pytest.mark.asyncio +async def test_priority_generic_fallback_keeps_routing_controls_out_of_provider_kwargs( + endpoint_request: dict[str, object], +) -> None: + async def provider(model: str, **provider_kwargs: object) -> str: + assert "_target_order" not in provider_kwargs + assert "model_priorities" not in provider_kwargs + model_info: Final = provider_kwargs["model_info"] + assert isinstance(model_info, dict) + if model_info["id"] != "deploy-3": + raise litellm.RateLimitError(message="primary refused", model=model, llm_provider="openai") + return "backup" + + router: Final = Router( + model_list=_priority_deployments(), + routing_groups=[_priority_group()], + num_retries=0, + disable_cooldowns=True, + ) + response: Final = await router._ageneric_api_call_with_fallbacks( + model="priority-group", original_function=provider, **endpoint_request + ) + assert response == "backup" + + +@pytest.mark.parametrize("provider_model_id", [None, "resolved-downstream-id"]) +@pytest.mark.asyncio +async def test_priority_generic_fallback_dict_preserves_served_model_id(provider_model_id: str | None) -> None: + async def provider(model: str, **provider_kwargs: object) -> dict[str, object]: + model_info: Final = provider_kwargs["model_info"] + assert isinstance(model_info, dict) + if model_info["id"] == "deploy-1": + raise litellm.NotFoundError(message="primary missing", model=model, llm_provider="openai") + assert model_info["id"] == "deploy-3" + return { + "content": "backup", + **({"_hidden_params": {"model_id": provider_model_id}} if provider_model_id is not None else {}), + } + + router: Final = Router( + model_list=_priority_deployments()[::2], + routing_groups=[_priority_group()], + num_retries=0, + disable_cooldowns=True, + ) + outer_metadata: Final[dict[str, object]] = {} + response: Final = await router._ageneric_api_call_with_fallbacks( + model="priority-group", + original_function=provider, + messages=[{"role": "user", "content": "hi"}], + litellm_metadata=outer_metadata, + ) + primary_info: Final = outer_metadata["model_info"] + assert isinstance(primary_info, dict) + assert primary_info["id"] == "deploy-1" + assert response["content"] == "backup" + assert response["_hidden_params"]["model_id"] == (provider_model_id or "deploy-3") + assert response["_hidden_params"]["additional_headers"]["x-litellm-attempted-fallbacks"] == 1 + + +@pytest.mark.asyncio +async def test_priority_group_ignores_cached_backup_affinity() -> None: + from litellm.router_utils.prompt_caching_cache import PromptCachingCache + + router: Final = Router( + model_list=_priority_deployments(), + routing_groups=[_priority_group()], + optional_pre_call_checks=["prompt_caching"], + ) + messages: Final = [{ + "role": "user", + "content": [{"type": "text", "text": "word " * 5000, "cache_control": {"type": "ephemeral"}}], + }] + cache: Final = PromptCachingCache(cache=router.cache) + await cache.async_add_model_id(model_id="deploy-3", messages=messages, tools=None) + assert await cache.async_get_model_id(messages=messages, tools=None) == {"model_id": "deploy-3"} + response: Final = await router.acompletion(model="priority-group", messages=messages) + assert response.choices[0].message.content == "primary" + + +@pytest.mark.asyncio +async def test_priority_group_preserves_responses_continuity_on_a_backup() -> None: + from litellm.responses.utils import ResponsesAPIRequestUtils + + router: Final = Router( + model_list=_priority_deployments(), + routing_groups=[_priority_group()], + optional_pre_call_checks=["responses_api_deployment_check"], + ) + previous_response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", model_id="deploy-3", response_id="resp-prior" + ) + deployment: Final = await router.async_get_available_deployment( + model="priority-group", input="continue", request_kwargs={"previous_response_id": previous_response_id} + ) + assert deployment["model_info"]["id"] == "deploy-3" + + +def _recording_fallback_gate(allowed: frozenset[str], checked: list[str]) -> FallbackAccessCheck: + async def check(*, model: str, request_kwargs: Mapping[str, object], llm_router: Router) -> bool: + checked.append(model) + return model in allowed + + return check + + +@pytest.mark.parametrize("budget_admits_group", [False, True]) +@pytest.mark.asyncio +async def test_auto_priority_advance_preserves_access_scope_and_checks_paid_group_budget( + budget_admits_group: bool, +) -> None: + access_checks: Final[list[str]] = [] + budget_checks: Final[list[str]] = [] + router: Final = Router( + model_list=[ + *_priority_deployments(primary_response="litellm.RateLimitError"), + _priority_auto_router_deployment(), + ], + routing_groups=[_priority_group()], + num_retries=0, + disable_cooldowns=True, + fallback_access_check=_recording_fallback_gate(frozenset({"smart-router"}), access_checks), + fallback_budget_check=_recording_fallback_gate( + frozenset({"priority-group"}) if budget_admits_group else frozenset(), budget_checks + ), + ) + if budget_admits_group: + response: Final = await router.acompletion( + model="smart-router", messages=[{"role": "user", "content": "hi"}] + ) + assert response.choices[0].message.content == "backup" + else: + with pytest.raises(litellm.RateLimitError): + await router.acompletion(model="smart-router", messages=[{"role": "user", "content": "hi"}]) + assert access_checks == [] + assert budget_checks == ["priority-group"] + + +@pytest.mark.asyncio +async def test_auto_priority_group_exhaustion_still_checks_external_fallback_access() -> None: + access_checks: Final[list[str]] = [] + router: Final = Router( + model_list=[ + *_priority_deployments( + primary_response="litellm.RateLimitError", backup_response="litellm.RateLimitError" + ), + _priority_auto_router_deployment(), + { + **_model_list()[2], + "model_name": "external", + "litellm_params": {**_model_list()[2]["litellm_params"], "mock_response": "external"}, + "model_info": {"id": "external-deployment"}, + }, + ], + routing_groups=[_priority_group()], + fallbacks=[{"priority-group": ["external"]}], + num_retries=0, + disable_cooldowns=True, + fallback_access_check=_recording_fallback_gate(frozenset({"smart-router"}), access_checks), + ) + with pytest.raises(litellm.RateLimitError): + await router.acompletion(model="smart-router", messages=[{"role": "user", "content": "hi"}]) + assert access_checks and set(access_checks) == {"external"} + + +@pytest.mark.parametrize("check_kind", ["access", "budget"]) +@pytest.mark.parametrize("metadata_bucket", ["metadata", "litellm_metadata"]) +@pytest.mark.asyncio +async def test_caller_cannot_spoof_a_priority_group_to_bypass_fallback_gates( + check_kind: Literal["access", "budget"], metadata_bucket: str, +) -> None: + checked: Final[list[str]] = [] + check: Final = _recording_fallback_gate(frozenset({"filtered-model"}), checked) + router: Final = Router( + model_list=_priority_deployments(primary_response="litellm.RateLimitError"), + routing_groups=[_priority_group()], + fallbacks=[{"filtered-model": ["priority-group"]}], + num_retries=0, + disable_cooldowns=True, + fallback_access_check=check if check_kind == "access" else None, + fallback_budget_check=check if check_kind == "budget" else None, + ) + with pytest.raises(litellm.RateLimitError): + await router.acompletion( + model="filtered-model", + messages=[{"role": "user", "content": "hi"}], + **{metadata_bucket: {"pre_routing_selected_model": "priority-group"}}, + ) + assert checked == ["priority-group"] diff --git a/tests/test_litellm/router_utils/test_access_windows.py b/tests/test_litellm/router_utils/test_access_windows.py new file mode 100644 index 00000000000..a797a788b4e --- /dev/null +++ b/tests/test_litellm/router_utils/test_access_windows.py @@ -0,0 +1,190 @@ +from datetime import datetime, time, timezone +from typing import Final + + +from litellm.router_utils.access_windows import ( + access_windows_config_error, + filter_reserved_deployments, + is_window_active, +) +from litellm.types.router import ModelAccessWindow + +_NIGHT_NY: Final = ModelAccessWindow( + start=time(22, 0), + end=time(6, 0), + timezone="America/New_York", + team_ids=("team-nightly",), +) + + +def _window(start: str, end: str, tz: str = "UTC", team_ids=("team-a",)) -> ModelAccessWindow: + return ModelAccessWindow( + start=time.fromisoformat(start), + end=time.fromisoformat(end), + timezone=tz, + team_ids=tuple(team_ids), + ) + + +def _deployment(windows: object = None) -> dict: + if windows is None: + return {"model_info": {}} + return {"model_info": {"access_windows": windows}} + + +def test_same_day_window_active_and_inactive(): + window: Final = _window("09:00", "17:00") + assert is_window_active(window, datetime(2026, 3, 9, 12, 0, tzinfo=timezone.utc)) is True + assert is_window_active(window, datetime(2026, 3, 9, 20, 0, tzinfo=timezone.utc)) is False + + +def test_cross_midnight_window(): + window: Final = _window("22:00", "06:00") + assert is_window_active(window, datetime(2026, 3, 9, 23, 0, tzinfo=timezone.utc)) is True + assert is_window_active(window, datetime(2026, 3, 10, 5, 59, tzinfo=timezone.utc)) is True + assert is_window_active(window, datetime(2026, 3, 9, 12, 0, tzinfo=timezone.utc)) is False + + +def test_start_boundary_inclusive_and_end_boundary_exclusive(): + window: Final = _window("22:00", "06:00") + assert is_window_active(window, datetime(2026, 3, 9, 22, 0, tzinfo=timezone.utc)) is True + assert is_window_active(window, datetime(2026, 3, 10, 6, 0, tzinfo=timezone.utc)) is False + + +def test_dst_spring_forward_gap_uses_real_local_time(): + window: Final = ModelAccessWindow( + start=time(1, 30), + end=time(3, 30), + timezone="America/New_York", + team_ids=("team-a",), + ) + assert is_window_active(window, datetime(2026, 3, 8, 6, 30, tzinfo=timezone.utc)) is True + assert is_window_active(window, datetime(2026, 3, 8, 7, 0, tzinfo=timezone.utc)) is True + assert is_window_active(window, datetime(2026, 3, 8, 7, 30, tzinfo=timezone.utc)) is False + + +def test_naive_now_is_treated_as_utc(): + window: Final = _window("09:00", "17:00") + assert is_window_active(window, datetime(2026, 3, 9, 12, 0)) is True + + +def test_team_in_second_window_is_kept(): + deployments: Final = ( + _deployment([ + {"start": "01:00", "end": "02:00", "timezone": "UTC", "team_ids": ["team-other"]}, + {"start": "20:00", "end": "23:59", "timezone": "UTC", "team_ids": ["team-a"]}, + ]), + ) + result: Final = filter_reserved_deployments( + deployments, "team-a", now=datetime(2026, 3, 9, 21, 0, tzinfo=timezone.utc) + ) + assert result.deployments == deployments + assert result.blocking_window is None + + +def test_unlisted_team_is_dropped_with_blocking_window(): + deployments: Final = (_deployment([_NIGHT_NY.model_dump()]),) + result: Final = filter_reserved_deployments( + deployments, "team-b", now=datetime(2026, 3, 10, 4, 0, tzinfo=timezone.utc) + ) + assert result.deployments == () + assert result.blocking_window == _NIGHT_NY + + +def test_missing_team_id_is_dropped(): + result: Final = filter_reserved_deployments( + (_deployment([_NIGHT_NY.model_dump()]),), + None, + now=datetime(2026, 3, 10, 4, 0, tzinfo=timezone.utc), + ) + assert result.deployments == () + assert result.blocking_window == _NIGHT_NY + + +def test_listed_team_is_kept(): + deployments: Final = (_deployment([_NIGHT_NY.model_dump()]),) + result: Final = filter_reserved_deployments( + deployments, "team-nightly", now=datetime(2026, 3, 10, 4, 0, tzinfo=timezone.utc) + ) + assert result.deployments == deployments + assert result.blocking_window is None + + +def test_deployment_without_windows_kept_for_anyone(): + deployments: Final = (_deployment(),) + result: Final = filter_reserved_deployments( + deployments, None, now=datetime(2026, 3, 10, 4, 0, tzinfo=timezone.utc) + ) + assert result.deployments == deployments + assert result.blocking_window is None + + +def test_inactive_window_keeps_deployment_for_unlisted_team(): + deployments: Final = (_deployment([_NIGHT_NY.model_dump()]),) + result: Final = filter_reserved_deployments( + deployments, "team-b", now=datetime(2026, 3, 10, 16, 0, tzinfo=timezone.utc) + ) + assert result.deployments == deployments + assert result.blocking_window is None + + +def test_unreserved_deployment_survives_for_other_team(): + reserved: Final = _deployment([_NIGHT_NY.model_dump()]) + open_deployment: Final = _deployment() + result: Final = filter_reserved_deployments( + (reserved, open_deployment), + "team-b", + now=datetime(2026, 3, 10, 4, 0, tzinfo=timezone.utc), + ) + assert result.deployments == (open_deployment,) + assert result.blocking_window == _NIGHT_NY + + +def test_config_error_unknown_timezone(): + error: Final = access_windows_config_error( + {"access_windows": [{"start": "22:00", "end": "06:00", "timezone": "Mars/Olympus", "team_ids": ["t"]}]}, + model_name="nightly-model", + ) + assert error is not None + assert "nightly-model" in error + assert "access_windows" in error + assert "Mars/Olympus" in error + + +def test_config_error_bad_time(): + error: Final = access_windows_config_error( + {"access_windows": [{"start": "25:00", "end": "06:00", "timezone": "UTC", "team_ids": ["t"]}]}, + model_name="m", + ) + assert error is not None + assert "access_windows" in error + + +def test_config_error_empty_team_ids(): + error: Final = access_windows_config_error( + {"access_windows": [{"start": "22:00", "end": "06:00", "timezone": "UTC", "team_ids": []}]}, + model_name="m", + ) + assert error is not None + + +def test_config_error_start_equals_end(): + error: Final = access_windows_config_error( + {"access_windows": [{"start": "22:00", "end": "22:00", "timezone": "UTC", "team_ids": ["t"]}]}, + model_name="m", + ) + assert error is not None + + +def test_config_error_none_when_absent(): + assert access_windows_config_error({}, model_name="m") is None + assert access_windows_config_error({"access_windows": None}, model_name="m") is None + + +def test_config_error_offset_aware_time(): + error: Final = access_windows_config_error( + {"access_windows": [{"start": "22:00+05:00", "end": "06:00", "timezone": "UTC", "team_ids": ["t"]}]}, + model_name="m", + ) + assert error is not None + assert "UTC offset" in error diff --git a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py index adee44aa8a3..4617839c5e3 100644 --- a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py +++ b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py @@ -418,6 +418,25 @@ class TestGpt6AstraAdvertisesItsDocumentedLevels: ) +class TestGpt6SolAndLunaAdvertiseNoneThroughMax: + @pytest.mark.parametrize("model", ["gpt-6-sol", "gpt-6-luna"]) + def test_the_entry_advertises_none_through_max(self, local_model_cost_map, model): + """OpenAI documents none, low, medium (default), high, xhigh and max for both. Unlike + gpt-6-astra they take none.""" + from litellm.utils import _get_model_info_helper + + model_info = dict(_get_model_info_helper(model=model, custom_llm_provider="openai")) + + assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == ( + "none", + "low", + "medium", + "high", + "xhigh", + "max", + ) + + class TestNearestDeclaredReasoningEffort: def test_a_declared_level_is_kept(self): assert nearest_declared_reasoning_effort("high", ("none", "high")) == "high" diff --git a/tests/test_litellm/rust_bridge/ocr/test_secrets.py b/tests/test_litellm/rust_bridge/ocr/test_secrets.py new file mode 100644 index 00000000000..085a42dd373 --- /dev/null +++ b/tests/test_litellm/rust_bridge/ocr/test_secrets.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.integrations.custom_secret_manager import CustomSecretManager +from litellm.llms.base_llm.ocr.transformation import OCRResponse +from litellm.rust_bridge import configuration +from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem +from tests.test_litellm_rust.support.recording_server import ResponseSpec, recording_service + + +class _VaultSecrets(CustomSecretManager): + def __init__(self) -> None: + super().__init__(secret_manager_name="rust_bridge_ocr_test") + + async def async_read_secret( + self, + secret_name: str, + optional_params: dict[str, object] | None = None, + timeout: float | httpx.Timeout | None = None, + ) -> str | None: + return "vault-key" if secret_name == "MISTRAL_API_KEY" else None + + def sync_read_secret( + self, + secret_name: str, + optional_params: dict[str, object] | None = None, + timeout: float | httpx.Timeout | None = None, + ) -> str | None: + return "vault-key" if secret_name == "MISTRAL_API_KEY" else None + + +async def _call(asynchronous: bool, api_base: str) -> OCRResponse: + if asynchronous: + return await litellm.aocr( + model="mistral/mistral-ocr-latest", + document={"type": "document_url", "document_url": "https://example.com/document.pdf"}, + api_base=api_base, + ) + return litellm.ocr( + model="mistral/mistral-ocr-latest", + document={"type": "document_url", "document_url": "https://example.com/document.pdf"}, + api_base=api_base, + ) + + +_RESPONSE: Final = { + "pages": [{"index": 0, "markdown": "parsed document", "images": []}], + "model": "mistral-ocr-latest", + "usage_info": {"pages_processed": 1}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", (False, True)) +@pytest.mark.parametrize("rust_enabled", ("0", "1")) +@pytest.mark.parametrize("access_mode", ("read_only", "read_and_write")) +@pytest.mark.parametrize("system", (None, KeyManagementSystem.CUSTOM)) +async def test_readable_secret_managers_keep_python_ocr_fallback( + monkeypatch: pytest.MonkeyPatch, + asynchronous: bool, + rust_enabled: str, + access_mode: str, + system: KeyManagementSystem | None, +) -> None: + pytest.importorskip("litellm.rust_bridge._native") + monkeypatch.setenv("LITELLM_RUST", rust_enabled) + monkeypatch.setenv("MISTRAL_API_KEY", "environment-key") + monkeypatch.setattr(litellm, "secret_manager_client", _VaultSecrets()) + monkeypatch.setattr(litellm, "_key_management_system", system) + monkeypatch.setattr( + litellm, + "_key_management_settings", + KeyManagementSettings(access_mode=access_mode, hosted_keys=["MISTRAL_API_KEY"]), + ) + configuration.reset_rust_configuration() + + with recording_service() as server: + server.default_response = ResponseSpec(body=_RESPONSE) + result: Final = await _call(asynchronous, server.base_url) + + assert result.pages[0].markdown == "parsed document" + assert len(server.requests) == 1 + expected_key: Final = "vault-key" if system is KeyManagementSystem.CUSTOM else "environment-key" + assert server.requests[0].headers["authorization"] == f"Bearer {expected_key}" + assert "x-litellm-rust" not in result._hidden_params.get("additional_headers", {}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", (False, True)) +async def test_no_secret_client_leaves_dormant_binding_settings_unread( + monkeypatch: pytest.MonkeyPatch, asynchronous: bool +) -> None: + pytest.importorskip("litellm.rust_bridge._native") + monkeypatch.setenv("LITELLM_RUST", "1") + monkeypatch.setenv("MISTRAL_API_KEY", "environment-key") + monkeypatch.setattr(litellm, "secret_manager_client", None) + monkeypatch.setattr(litellm, "_key_management_settings", object()) + configuration.reset_rust_configuration() + + with recording_service() as server: + server.default_response = ResponseSpec(body=_RESPONSE) + result: Final = await _call(asynchronous, server.base_url) + + assert result.pages[0].markdown == "parsed document" + assert len(server.requests) == 1 + assert server.requests[0].headers["authorization"] == "Bearer environment-key" + assert result._hidden_params["additional_headers"]["x-litellm-rust"] == "true" diff --git a/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py b/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py index 1f6a214398a..05f2d13a079 100644 --- a/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py +++ b/tests/test_litellm/rust_bridge/test_callbacks_legacy_python.py @@ -9,9 +9,10 @@ import pytest from pydantic import TypeAdapter import litellm +from litellm._internal_context import is_internal_call from litellm.litellm_core_utils.litellm_logging import Logging from litellm.rust_bridge import callbacks_legacy_python as legacy -from litellm.rust_bridge.callbacks_legacy_python import check_limits, setup +from litellm.rust_bridge.callbacks_legacy_python import check_limits, failure_handler, setup _OCR_KWARGS: Final = MappingProxyType( { @@ -81,6 +82,82 @@ def test_setup_builds_a_logger_when_none_is_supplied(call_type: str, kwargs: Map assert result.logger.litellm_call_id == result.kwargs["litellm_call_id"] +def _budget_reservation() -> dict: + return {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": False} + + +def _kwargs_with_a_budget_reservation(reservation: dict) -> dict[str, object]: + return {**_OCR_KWARGS, "metadata": {"user_api_key_budget_reservation": reservation}} + + +def test_setup_claims_the_budget_reservation_for_an_async_call() -> None: + reservation: Final = _budget_reservation() + + setup("aocr", (), _kwargs_with_a_budget_reservation(reservation), datetime.datetime.now(), asynchronous=True) + + assert reservation["callback_bound"] is True + + +def test_setup_claims_the_budget_reservation_a_supplied_logger_already_saw() -> None: + reservation: Final = _budget_reservation() + supplied: Final = _supplied_logger() + supplied.update_environment_variables( + litellm_params={"metadata": {"user_api_key_budget_reservation": reservation}}, optional_params={} + ) + assert reservation["callback_bound"] is False + + setup("aocr", (), {**_OCR_KWARGS, "litellm_logging_obj": supplied}, datetime.datetime.now(), asynchronous=True) + + assert reservation["callback_bound"] is True + + +def test_setup_leaves_the_budget_reservation_alone_for_a_sync_call() -> None: + reservation: Final = _budget_reservation() + + setup("aocr", (), _kwargs_with_a_budget_reservation(reservation), datetime.datetime.now(), asynchronous=False) + + assert reservation["callback_bound"] is False + + +def test_setup_leaves_the_budget_reservation_alone_for_an_internal_call() -> None: + reservation: Final = _budget_reservation() + token: Final = is_internal_call.set(True) + try: + setup("aocr", (), _kwargs_with_a_budget_reservation(reservation), datetime.datetime.now(), asynchronous=True) + finally: + is_internal_call.reset(token) + + assert reservation["callback_bound"] is False + + +def test_failure_handler_hands_the_budget_reservation_back_for_an_async_call() -> None: + reservation: Final = _budget_reservation() + now: Final = datetime.datetime.now() + result: Final = setup("aocr", (), _kwargs_with_a_budget_reservation(reservation), now, asynchronous=True) + assert reservation["callback_bound"] is True + + pending: Final = failure_handler(result.logger, RuntimeError("upstream refused"), now, now, asynchronous=True) + + assert reservation["callback_bound"] is False + assert pending is not None + pending.close() + + +def test_failure_handler_of_an_internal_call_leaves_the_outer_budget_reservation_claim_in_place() -> None: + reservation: Final = _budget_reservation() + now: Final = datetime.datetime.now() + result: Final = setup("aocr", (), _kwargs_with_a_budget_reservation(reservation), now, asynchronous=True) + token: Final = is_internal_call.set(True) + try: + pending: Final = failure_handler(result.logger, RuntimeError("inner step failed"), now, now, asynchronous=True) + finally: + is_internal_call.reset(token) + + assert reservation["callback_bound"] is True + assert pending is not None + pending.close() + + CONTRACT_PATH: Final = ( Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json" ) diff --git a/tests/test_litellm/rust_bridge/test_catalog.py b/tests/test_litellm/rust_bridge/test_catalog.py index 147e863baf5..b6b486e73f0 100644 --- a/tests/test_litellm/rust_bridge/test_catalog.py +++ b/tests/test_litellm/rust_bridge/test_catalog.py @@ -6,8 +6,21 @@ from typing import Final import pytest from litellm.rust_bridge import catalog, configuration -from litellm.rust_bridge.catalog import Context, Delivery, Route, Rule +from litellm.rust_bridge.catalog import ( + CacheContext, + CacheRule, + Context, + Delivery, + Route, + RouteContext, + RouteRule, + Rules, + SecretManagerContext, + SecretManagerRule, +) from litellm.rust_bridge.configuration import Decision, Rollout +from litellm.types.caching import LiteLLMCacheType +from litellm.types.secret_managers.main import KeyManagementSystem @pytest.fixture(autouse=True) @@ -34,16 +47,12 @@ def test_shipped_decisions( configuration.rust(process) if environment is not None: monkeypatch.setenv("LITELLM_RUST", environment) - context: Final = Context(route, provider=provider, model="test-model", delivery=delivery) + context: Final = RouteContext(route, provider=provider, model="test-model", delivery=delivery) if route is Route.OCR: enabled: Final = environment == "1" if environment is not None else process is not False assert catalog.rollout(context) is Rollout.RUST_OPT_OUT assert catalog.decision(context) is (Decision.RUST_WITH_FALLBACK if enabled else Decision.PYTHON) - elif route is Route.MESSAGES: - enabled: Final = environment == "1" if environment is not None else process is True - assert catalog.rollout(context) is Rollout.RUST_OPT_IN - assert catalog.decision(context) is (Decision.RUST_WITH_FALLBACK if enabled else Decision.PYTHON) elif route is Route.TRANSCRIPTION and provider == "bedrock": assert catalog.rollout(context) is Rollout.RUST_REQUIRED assert catalog.decision(context) is Decision.RUST_REQUIRED @@ -57,31 +66,63 @@ def test_missing_rule_stays_on_python_even_when_rust_is_enabled(monkeypatch: pyt configuration.rust(True) monkeypatch.setenv("LITELLM_RUST", "1") - assert catalog.rollout(Context(route), rules=()) is Rollout.PYTHON_ONLY - assert catalog.decision(Context(route), rules=()) is Decision.PYTHON + assert catalog.rollout(RouteContext(route), rules=()) is Rollout.PYTHON_ONLY + assert catalog.decision(RouteContext(route), rules=()) is Decision.PYTHON + + +@pytest.mark.parametrize( + "context", + ( + *(CacheContext(backend.value) for backend in LiteLLMCacheType), + *(SecretManagerContext(system.value) for system in KeyManagementSystem), + CacheContext("custom"), + SecretManagerContext("unknown"), + ), +) +def test_backend_rollouts_stay_on_python_when_global_rust_is_enabled( + monkeypatch: pytest.MonkeyPatch, context: Context +) -> None: + configuration.rust(True) + monkeypatch.setenv("LITELLM_RUST", "1") + + assert catalog.rollout(context) is Rollout.PYTHON_ONLY + assert catalog.decision(context) is Decision.PYTHON + + +def test_response_cache_rules_select_the_whole_backend_runtime() -> None: + rules: Final = ( + CacheRule(Rollout.RUST_REQUIRED, backends=frozenset({"local"})), + CacheRule(Rollout.PYTHON_ONLY), + ) + + assert catalog.decision(CacheContext(backend="local"), rules) is Decision.RUST_REQUIRED + assert catalog.decision(CacheContext(backend="redis"), rules) is Decision.PYTHON @pytest.mark.parametrize( ("context", "expected"), ( - (Context(Route.RESPONSES, provider="openai", model="m", delivery=Delivery.WEBSOCKET), Decision.RUST_REQUIRED), - (Context(Route.RESPONSES, provider="openai", model="m"), Decision.PYTHON), - (Context(Route.RESPONSES, provider="openai", model="m", delivery=Delivery.STREAMING), Decision.PYTHON), - (Context(Route.RESPONSES, provider="openai", model="other", delivery=Delivery.WEBSOCKET), Decision.PYTHON), - (Context(Route.RESPONSES, provider="anthropic", model="m", delivery=Delivery.WEBSOCKET), Decision.PYTHON), - (Context(Route.MESSAGES, provider="openai", model="m", delivery=Delivery.WEBSOCKET), Decision.PYTHON), + ( + RouteContext(Route.RESPONSES, provider="openai", model="m", delivery=Delivery.WEBSOCKET), + Decision.RUST_REQUIRED, + ), + (RouteContext(Route.RESPONSES, provider="openai", model="m"), Decision.PYTHON), + (RouteContext(Route.RESPONSES, provider="openai", model="m", delivery=Delivery.STREAMING), Decision.PYTHON), + (RouteContext(Route.RESPONSES, provider="openai", model="other", delivery=Delivery.WEBSOCKET), Decision.PYTHON), + (RouteContext(Route.RESPONSES, provider="anthropic", model="m", delivery=Delivery.WEBSOCKET), Decision.PYTHON), + (RouteContext(Route.MESSAGES, provider="openai", model="m", delivery=Delivery.WEBSOCKET), Decision.PYTHON), ), ) -def test_first_matching_rule_respects_every_constraint(context: Context, expected: Decision) -> None: +def test_first_matching_rule_respects_every_constraint(context: RouteContext, expected: Decision) -> None: rules: Final = ( - Rule( + RouteRule( Route.RESPONSES, Rollout.RUST_REQUIRED, providers=frozenset({"openai"}), models=frozenset({"m"}), deliveries=frozenset({Delivery.WEBSOCKET}), ), - Rule(Route.RESPONSES, Rollout.PYTHON_ONLY), + RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY), ) assert catalog.decision(context, rules) is expected @@ -96,4 +137,78 @@ def test_textract_ocr_has_no_python_path_to_opt_out_to( if environment is not None: monkeypatch.setenv("LITELLM_RUST", environment) - assert catalog.decision(Context(Route.OCR, provider="aws_textract", model="m")) is Decision.RUST_REQUIRED + assert catalog.decision(RouteContext(Route.OCR, provider="aws_textract", model="m")) is Decision.RUST_REQUIRED + + +@pytest.mark.parametrize( + ("context", "expected"), + ( + (RouteContext(Route.OCR, provider="local"), Decision.RUST_REQUIRED), + (RouteContext(Route.OCR, provider="other"), Decision.PYTHON), + (RouteContext(Route.MESSAGES, provider="local"), Decision.PYTHON), + (CacheContext("local"), Decision.RUST_WITH_FALLBACK), + (CacheContext("other"), Decision.PYTHON), + (SecretManagerContext("local"), Decision.PYTHON), + (SecretManagerContext("other"), Decision.RUST_REQUIRED), + ), +) +def test_mixed_rules_select_only_the_matching_domain(context: Context, expected: Decision) -> None: + rules: Final[Rules] = ( + CacheRule(Rollout.RUST_OPT_OUT, backends=frozenset({"local"})), + CacheRule(Rollout.PYTHON_ONLY), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({"local"})), + SecretManagerRule(Rollout.RUST_REQUIRED), + RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"local"})), + RouteRule(Route.OCR, Rollout.PYTHON_ONLY), + ) + + assert catalog.decision(context, rules) is expected + + +@pytest.mark.parametrize("context", (RouteContext(Route.OCR), CacheContext("local"), SecretManagerContext("local"))) +@pytest.mark.parametrize( + ("rollout", "process", "environment", "expected"), + ( + (Rollout.PYTHON_ONLY, True, "1", Decision.PYTHON), + (Rollout.RUST_REQUIRED, False, "0", Decision.RUST_REQUIRED), + (Rollout.RUST_OPT_IN, None, None, Decision.PYTHON), + (Rollout.RUST_OPT_OUT, None, None, Decision.RUST_WITH_FALLBACK), + (Rollout.RUST_OPT_IN, True, None, Decision.RUST_WITH_FALLBACK), + (Rollout.RUST_OPT_OUT, False, None, Decision.PYTHON), + (Rollout.RUST_OPT_IN, False, "1", Decision.RUST_WITH_FALLBACK), + (Rollout.RUST_OPT_OUT, True, "0", Decision.PYTHON), + ), +) +def test_all_domains_share_rollout_switches_and_first_match( + monkeypatch: pytest.MonkeyPatch, + context: Context, + rollout: Rollout, + process: bool | None, + environment: str | None, + expected: Decision, +) -> None: + configuration.rust(process) + if environment is not None: + monkeypatch.setenv("LITELLM_RUST", environment) + rules: Final[Rules] = ( + RouteRule(Route.OCR, rollout), + CacheRule(rollout), + SecretManagerRule(rollout), + RouteRule(Route.OCR, Rollout.RUST_REQUIRED), + CacheRule(Rollout.RUST_REQUIRED), + SecretManagerRule(Rollout.RUST_REQUIRED), + ) + + assert catalog.decision(context, rules) is expected + assert catalog.decision(context, ()) is Decision.PYTHON + + +@pytest.mark.parametrize("context", (RouteContext(Route.OCR), CacheContext("local"), SecretManagerContext("local"))) +def test_empty_constraints_match_nothing(context: Context) -> None: + rules: Final[Rules] = ( + RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset()), + CacheRule(Rollout.RUST_REQUIRED, backends=frozenset()), + SecretManagerRule(Rollout.RUST_REQUIRED, systems=frozenset()), + ) + + assert catalog.decision(context, rules) is Decision.PYTHON diff --git a/tests/test_litellm/rust_bridge/test_dispatch.py b/tests/test_litellm/rust_bridge/test_dispatch.py index 66f8d114f7a..9a3793a772e 100644 --- a/tests/test_litellm/rust_bridge/test_dispatch.py +++ b/tests/test_litellm/rust_bridge/test_dispatch.py @@ -6,7 +6,7 @@ import pytest from litellm.rust_bridge import configuration from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Context, Delivery, Route, Rule, Rules +from litellm.rust_bridge.catalog import CacheRule, Delivery, Route, RouteContext, RouteRule, Rules, SecretManagerRule from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.dispatch import PublicDispatch @@ -22,14 +22,15 @@ def binding() -> NativeBinding[object]: return bound -def test_route_without_rules_forwards_before_request_projection() -> None: +@pytest.mark.parametrize("rules", ((), (CacheRule(Rollout.RUST_REQUIRED), SecretManagerRule(Rollout.RUST_REQUIRED)))) +def test_route_without_rules_forwards_before_request_projection(rules: Rules) -> None: stream: Final[Iterator[int]] = iter((1, 2)) def reject_request(args: tuple[object, ...], kwargs: Mapping[str, object]) -> Request: pytest.fail("Python-only routes must not project the request") dispatch: Final = PublicDispatch( - route=Route.CHAT_COMPLETIONS, request=reject_request, context=lambda _: Context(Route.CHAT_COMPLETIONS) + route=Route.CHAT_COMPLETIONS, request=reject_request, context=lambda _: RouteContext(Route.CHAT_COMPLETIONS) ) result: Final = dispatch.run( ("model",), @@ -37,15 +38,15 @@ def test_route_without_rules_forwards_before_request_projection() -> None: python=lambda *args, **kwargs: stream, binding=binding(), native=lambda hook, request, args, kwargs: pytest.fail("Python-only routes must not call native"), - rules=(), + rules=rules, ) assert result is stream def test_unconditional_python_rule_prevents_later_rust_rule_projection() -> None: rules: Final[Rules] = ( - Rule(Route.CHAT_COMPLETIONS, Rollout.PYTHON_ONLY), - Rule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED), + RouteRule(Route.CHAT_COMPLETIONS, Rollout.PYTHON_ONLY), + RouteRule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED), ) def reject_request(args: tuple[object, ...], kwargs: Mapping[str, object]) -> Request: @@ -54,7 +55,7 @@ def test_unconditional_python_rule_prevents_later_rust_rule_projection() -> None dispatch: Final = PublicDispatch( route=Route.CHAT_COMPLETIONS, request=reject_request, - context=lambda _: Context(Route.CHAT_COMPLETIONS), + context=lambda _: RouteContext(Route.CHAT_COMPLETIONS), ) expected: Final = object() result: Final = dispatch.run( @@ -69,12 +70,12 @@ def test_unconditional_python_rule_prevents_later_rust_rule_projection() -> None def test_disabled_optional_rust_rule_forwards_before_projection() -> None: - rules: Final[Rules] = (Rule(Route.OCR, Rollout.RUST_OPT_OUT),) + rules: Final[Rules] = (RouteRule(Route.OCR, Rollout.RUST_OPT_OUT),) def reject_request(args: tuple[object, ...], kwargs: Mapping[str, object]) -> Request: pytest.fail("Disabled optional Rust must not project the request") - dispatch: Final = PublicDispatch(route=Route.OCR, request=reject_request, context=lambda _: Context(Route.OCR)) + dispatch: Final = PublicDispatch(route=Route.OCR, request=reject_request, context=lambda _: RouteContext(Route.OCR)) expected: Final = object() configuration.rust(False) try: @@ -95,12 +96,14 @@ def test_native_stream_result_is_not_consumed_or_wrapped() -> None: request: Final = Request(model="streaming-model") stream: Final[Iterator[int]] = iter((1, 2)) rules: Final[Rules] = ( - Rule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED, deliveries=frozenset({Delivery.STREAMING})), + CacheRule(Rollout.PYTHON_ONLY), + SecretManagerRule(Rollout.PYTHON_ONLY), + RouteRule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED, deliveries=frozenset({Delivery.STREAMING})), ) dispatch: Final = PublicDispatch( route=Route.CHAT_COMPLETIONS, request=lambda args, kwargs: request, - context=lambda value: Context(Route.CHAT_COMPLETIONS, model=value.model, delivery=Delivery.STREAMING), + context=lambda value: RouteContext(Route.CHAT_COMPLETIONS, model=value.model, delivery=Delivery.STREAMING), ) def native(request: Request, args: tuple[object, ...], kwargs: Mapping[str, object]) -> Iterator[int]: @@ -122,7 +125,8 @@ def test_native_stream_result_is_not_consumed_or_wrapped() -> None: @pytest.mark.asyncio -async def test_async_route_without_rules_preserves_async_iterator_result() -> None: +@pytest.mark.parametrize("rules", ((), (CacheRule(Rollout.RUST_REQUIRED), SecretManagerRule(Rollout.RUST_REQUIRED)))) +async def test_async_route_without_rules_preserves_async_iterator_result(rules: Rules) -> None: async def chunks() -> AsyncGenerator[int, None]: yield 1 @@ -135,7 +139,7 @@ async def test_async_route_without_rules_preserves_async_iterator_result() -> No return stream dispatch: Final = PublicDispatch( - route=Route.RESPONSES, request=reject_request, context=lambda _: Context(Route.RESPONSES) + route=Route.RESPONSES, request=reject_request, context=lambda _: RouteContext(Route.RESPONSES) ) result: Final = await dispatch.arun( ("model",), @@ -143,7 +147,7 @@ async def test_async_route_without_rules_preserves_async_iterator_result() -> No python=python, binding=binding(), native=lambda hook, request, args, kwargs: pytest.fail("Python-only routes must not call native"), - rules=(), + rules=rules, ) assert result is stream await stream.aclose() @@ -152,11 +156,13 @@ async def test_async_route_without_rules_preserves_async_iterator_result() -> No @pytest.mark.asyncio async def test_async_dispatch_accepts_websocket_style_none_result() -> None: request: Final = Request(model="realtime-model") - rules: Final[Rules] = (Rule(Route.RESPONSES, Rollout.RUST_REQUIRED, deliveries=frozenset({Delivery.WEBSOCKET})),) + rules: Final[Rules] = ( + RouteRule(Route.RESPONSES, Rollout.RUST_REQUIRED, deliveries=frozenset({Delivery.WEBSOCKET})), + ) dispatch: Final = PublicDispatch( route=Route.RESPONSES, request=lambda args, kwargs: request, - context=lambda value: Context(Route.RESPONSES, model=value.model, delivery=Delivery.WEBSOCKET), + context=lambda value: RouteContext(Route.RESPONSES, model=value.model, delivery=Delivery.WEBSOCKET), ) async def python(*args: object, **kwargs: object) -> None: # kwargs-ok: public pass-through shape @@ -183,14 +189,14 @@ async def test_async_dispatch_accepts_websocket_style_none_result() -> None: def test_rules_for_other_routes_and_constrained_python_rules_skip_projection() -> None: rules: Final[Rules] = ( - Rule(Route.MESSAGES, Rollout.RUST_REQUIRED), - Rule(Route.OCR, Rollout.PYTHON_ONLY, providers=frozenset({"mistral"})), + RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), + RouteRule(Route.OCR, Rollout.PYTHON_ONLY, providers=frozenset({"mistral"})), ) def reject_request(args: tuple[object, ...], kwargs: Mapping[str, object]) -> Request: pytest.fail("Rules that cannot select Rust must not project the request") - dispatch: Final = PublicDispatch(route=Route.OCR, request=reject_request, context=lambda _: Context(Route.OCR)) + dispatch: Final = PublicDispatch(route=Route.OCR, request=reject_request, context=lambda _: RouteContext(Route.OCR)) expected: Final = object() result: Final = dispatch.run( ("model",), @@ -206,11 +212,11 @@ def test_rules_for_other_routes_and_constrained_python_rules_skip_projection() - @pytest.mark.asyncio async def test_async_bypass_forwards_to_python_without_native() -> None: request: Final = Request(model="bypassed-model") - rules: Final[Rules] = (Rule(Route.RESPONSES, Rollout.RUST_REQUIRED),) + rules: Final[Rules] = (RouteRule(Route.RESPONSES, Rollout.RUST_REQUIRED),) dispatch: Final = PublicDispatch( route=Route.RESPONSES, request=lambda args, kwargs: request, - context=lambda value: Context(Route.RESPONSES, model=value.model), + context=lambda value: RouteContext(Route.RESPONSES, model=value.model), bypass=lambda value: value.model == "bypassed-model", ) expected: Final = object() diff --git a/tests/test_litellm/rust_bridge/test_runtime.py b/tests/test_litellm/rust_bridge/test_runtime.py index fa6c0b30413..bff7ded3114 100644 --- a/tests/test_litellm/rust_bridge/test_runtime.py +++ b/tests/test_litellm/rust_bridge/test_runtime.py @@ -10,7 +10,7 @@ from litellm.exceptions import APIError from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict from litellm.rust_bridge import bindings, configuration, runtime -from litellm.rust_bridge.catalog import Context, Delivery, Route, Rule +from litellm.rust_bridge.catalog import Delivery, Route, RouteContext, RouteRule from litellm.rust_bridge.configuration import Rollout @@ -39,7 +39,7 @@ class NativeFn(Protocol): def __call__(self) -> str: ... -CONTEXT: Final = Context(Route.MESSAGES, provider="anthropic", model="model") +CONTEXT: Final = RouteContext(Route.MESSAGES, provider="anthropic", model="model") RUST: Final = "rust" PYTHON: Final = "python" @@ -50,8 +50,8 @@ def binding(native: NativeFn | None) -> bindings.NativeBinding[NativeFn]: return bound -def rules(rollout: Rollout) -> tuple[Rule, ...]: - return (Rule(Route.MESSAGES, rollout, providers=frozenset({"anthropic"})),) +def rules(rollout: Rollout) -> tuple[RouteRule, ...]: + return (RouteRule(Route.MESSAGES, rollout, providers=frozenset({"anthropic"})),) class Recorder: @@ -74,7 +74,7 @@ def recorder(native_effect: BaseException | None = None) -> Recorder: return Recorder(native_effect) -def run(rollout: Rollout, calls: Recorder, *, native_missing: bool = False, context: Context = CONTEXT) -> str: +def run(rollout: Rollout, calls: Recorder, *, native_missing: bool = False, context: RouteContext = CONTEXT) -> str: return runtime.run( context, binding=binding(None if native_missing else calls.rust), @@ -146,8 +146,8 @@ def test_context_outside_rule_stays_on_python() -> None: calls: Final = recorder() configuration.rust(True) - assert run(Rollout.RUST_REQUIRED, calls, context=Context(Route.MESSAGES, provider="openai")) == "python" - assert run(Rollout.RUST_REQUIRED, calls, context=Context(Route.RESPONSES, provider="anthropic")) == "python" + assert run(Rollout.RUST_REQUIRED, calls, context=RouteContext(Route.MESSAGES, provider="openai")) == "python" + assert run(Rollout.RUST_REQUIRED, calls, context=RouteContext(Route.RESPONSES, provider="anthropic")) == "python" assert calls.calls == (PYTHON, PYTHON) @@ -155,20 +155,20 @@ def test_context_outside_rule_stays_on_python() -> None: @pytest.mark.parametrize( "context", ( - Context(Route.CHAT_COMPLETIONS, provider="anthropic"), - Context(Route.CHAT_COMPLETIONS, provider="bedrock"), - Context(Route.RESPONSES, provider="openai"), - Context(Route.TRANSCRIPTION, provider="openai"), + RouteContext(Route.CHAT_COMPLETIONS, provider="anthropic"), + RouteContext(Route.CHAT_COMPLETIONS, provider="bedrock"), + RouteContext(Route.RESPONSES, provider="openai"), + RouteContext(Route.TRANSCRIPTION, provider="openai"), ), ) @pytest.mark.parametrize("delivery", tuple(Delivery)) async def test_shipped_python_routes_never_load_native( - monkeypatch: pytest.MonkeyPatch, context: Context, delivery: Delivery + monkeypatch: pytest.MonkeyPatch, context: RouteContext, delivery: Delivery ) -> None: monkeypatch.setenv("LITELLM_RUST", "1") configuration.rust(True) calls: Final = recorder() - request: Final = Context(context.route, provider=context.provider, delivery=delivery) + request: Final = RouteContext(context.route, provider=context.provider, delivery=delivery) def reject_load(value: object) -> NativeFn | None: pytest.fail("Python-only dispatch must not load a native binding") diff --git a/tests/test_litellm/rust_bridge/test_settings.py b/tests/test_litellm/rust_bridge/test_settings.py index 023f02cffbb..3a86a69ed8b 100644 --- a/tests/test_litellm/rust_bridge/test_settings.py +++ b/tests/test_litellm/rust_bridge/test_settings.py @@ -1,12 +1,8 @@ -import dataclasses import logging -from pathlib import Path from typing import Final import httpx import pytest -from pydantic import TypeAdapter -from typing_extensions import ReadOnly, TypedDict import litellm from litellm.integrations.custom_secret_manager import CustomSecretManager @@ -15,34 +11,6 @@ from litellm.rust_bridge import settings from litellm.secret_managers.main import get_secret_str from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem -CONTRACT_PATH: Final = Path(__file__).parents[3] / "litellm-rust/crates/python-bridge/python_settings.json" - - -class SettingSpec(TypedDict): - adapter: ReadOnly[str] - required: ReadOnly[bool] - precedence: ReadOnly[str] - sensitive: ReadOnly[bool] - shapes: ReadOnly[list[str]] - unsupported_live: ReadOnly[str | None] - - -class SettingsGroup(TypedDict): - version: ReadOnly[int] - fields: ReadOnly[dict[str, SettingSpec]] - - -def test_the_rust_contract_matches_the_returned_fields() -> None: - contract: Final = TypeAdapter(dict[str, SettingsGroup]).validate_json(CONTRACT_PATH.read_text()) - - assert {name: tuple(group["fields"]) for name, group in contract.items()} == { - "http_settings": tuple(field.name for field in dataclasses.fields(settings.http_settings())), - "url_policy": tuple(field.name for field in dataclasses.fields(settings.url_policy())), - "provider_defaults": tuple(field.name for field in dataclasses.fields(settings.provider_defaults())), - "secret_manager": tuple(field.name for field in dataclasses.fields(settings.secret_manager())), - } - - def test_url_policy_reads_the_litellm_globals(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "user_url_validation", False) monkeypatch.setattr(litellm, "user_url_allowed_hosts", ["docs.internal:8443"]) @@ -140,6 +108,75 @@ def test_secret_manager_is_not_readable_without_a_client(monkeypatch: pytest.Mon assert settings.secret_manager() == settings.SecretManager(readable=False) +def test_secret_manager_projects_custom_settings(monkeypatch: pytest.MonkeyPatch) -> None: + manager_settings: Final = KeyManagementSettings( + access_mode="read_and_write", + hosted_keys=["MISTRAL_API_KEY"], + primary_secret_name="primary", + aws_region_name="us-east-1", + ) + client: Final = _VaultSecrets({"MISTRAL_API_KEY": "vault-key"}) + monkeypatch.setattr(litellm, "secret_manager_client", client) + monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM) + monkeypatch.setattr(litellm, "_key_management_settings", manager_settings) + + assert settings.secret_manager_binding() == settings.SecretManagerBinding( + system="custom", + access_mode="read_and_write", + hosted_keys=["MISTRAL_API_KEY"], + primary_secret_name="primary", + store_virtual_keys=manager_settings.store_virtual_keys, + prefix_for_stored_virtual_keys=manager_settings.prefix_for_stored_virtual_keys, + kms_key_id=manager_settings.kms_key_id, + custom_secret_manager=manager_settings.custom_secret_manager, + aws_region_name="us-east-1", + aws_role_name=manager_settings.aws_role_name, + aws_session_name=manager_settings.aws_session_name, + aws_external_id=manager_settings.aws_external_id, + aws_profile_name=manager_settings.aws_profile_name, + aws_web_identity_token=manager_settings.aws_web_identity_token, + aws_sts_endpoint=manager_settings.aws_sts_endpoint, + replica_regions=manager_settings.replica_regions, + client=client, + settings_object=manager_settings, + ) + + +def test_secret_manager_without_a_client_has_no_system(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "secret_manager_client", None) + + assert settings.secret_manager_binding().system is None + + +def test_secret_manager_uses_key_management_defaults(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "secret_manager_client", None) + monkeypatch.setattr(litellm, "_key_management_settings", None) + + defaults: Final = KeyManagementSettings() + result: Final = settings.secret_manager_binding() + + assert result == settings.SecretManagerBinding( + system=None, + access_mode=defaults.access_mode, + hosted_keys=defaults.hosted_keys, + primary_secret_name=defaults.primary_secret_name, + store_virtual_keys=defaults.store_virtual_keys, + prefix_for_stored_virtual_keys=defaults.prefix_for_stored_virtual_keys, + kms_key_id=defaults.kms_key_id, + custom_secret_manager=defaults.custom_secret_manager, + aws_region_name=defaults.aws_region_name, + aws_role_name=defaults.aws_role_name, + aws_session_name=defaults.aws_session_name, + aws_external_id=defaults.aws_external_id, + aws_profile_name=defaults.aws_profile_name, + aws_web_identity_token=defaults.aws_web_identity_token, + aws_sts_endpoint=defaults.aws_sts_endpoint, + replica_regions=defaults.replica_regions, + client=None, + settings_object=None, + ) + + def test_provider_defaults_read_the_litellm_globals(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "vertex_project", "configured-project") monkeypatch.setattr(litellm, "vertex_location", "europe-west4") diff --git a/tests/test_litellm/rust_bridge/test_token_counter.py b/tests/test_litellm/rust_bridge/test_token_counter.py index 71aa79cc4bb..3af7127a9b3 100644 --- a/tests/test_litellm/rust_bridge/test_token_counter.py +++ b/tests/test_litellm/rust_bridge/test_token_counter.py @@ -1,8 +1,7 @@ -"""Tests for the Rust input token counter bridge. +"""Tests for the Rust input token counter bridge, called directly rather than through the route catalog. -The native factory is dependency-injected through ``TOKEN_COUNTER.override`` -so the fallback cases run without the compiled extension present. The parity -cases need the extension and are skipped when it is not built. +The factory is passed into ``native_count`` so the caching cases run without the compiled extension +present. The parity cases need the extension and are skipped when it is not built. """ from __future__ import annotations @@ -12,41 +11,43 @@ from types import MappingProxyType from typing import Final import pytest -import tiktoken -from tokenizers import Tokenizer import litellm from litellm.constants import TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS from litellm.litellm_core_utils.token_counter import openai_tokenizer_encoding -from litellm.proxy.spend_tracking.budget_reservation import _count_input_tokens -from litellm.rust_bridge import bindings, configuration +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens_for_model from litellm.rust_bridge import token_counter as bridge +from litellm.rust_bridge import tokenizer as tokenizer_dispatch +from litellm.rust_bridge._native import Tokenizer from litellm.utils import claude_json_str MODEL: Final = "claude-sonnet-4-5-20250929" CL100K_MODEL: Final = "gpt-4" O200K_MODEL: Final = "gpt-4o" +MODEL_BY_TOKENIZER: Final[MappingProxyType[bridge.RustTokenizer, str]] = MappingProxyType( + {"anthropic": MODEL, "cl100k_base": CL100K_MODEL, "o200k_base": O200K_MODEL} +) TOKENIZERS: Final[tuple[bridge.RustTokenizer, ...]] = ("anthropic", "cl100k_base", "o200k_base") -RANK_FILE_LINES: Final = MappingProxyType({"cl100k_base": 100_256, "o200k_base": 199_998}) BODY: Final = json.dumps({"model": MODEL, "messages": [{"role": "user", "content": "hello"}]}).encode() -class _FakeDeclined(Exception): - pass +def _counted(body: dict[str, object], model: str) -> tuple[bytes, dict[str, object]]: + raw: Final = json.dumps({**body, "model": model}).encode() + return raw, json.loads(raw) -class _FakeUpstream(Exception): - pass +class _FakeTokenizer: + """Stands in for one shared native `Tokenizer`; only its name identifies it.""" - -class _FakeNative: - RustBridgeDeclined = _FakeDeclined - RustUpstreamError = _FakeUpstream + def __init__(self, name: str, json: str | None = None) -> None: + self.name = name + self.json = json class _RecordingCounter: - def __init__(self, tokenizer_json: str) -> None: - self.tokenizer_json = tokenizer_json + def __init__(self, tokenizer: _FakeTokenizer, fast: bool) -> None: + self.tokenizer = tokenizer + self.fast = fast self.bodies: list[bytes] = [] async def acount_request(self, body: bytes) -> object: @@ -55,149 +56,81 @@ class _RecordingCounter: class _RecordingFactory: - """Stands in for the native `TokenCounter` class: callable for tokenizer JSON, `from_*_ranks` for rank files.""" + """Stands in for the native `TokenCounter` class, built over a loaded `Tokenizer`.""" def __init__(self) -> None: self.counters: list[_RecordingCounter] = [] - self.rank_files: list[str] = [] - def __call__(self, tokenizer_json: str) -> _RecordingCounter: - counter = _RecordingCounter(tokenizer_json) + def from_tokenizer(self, tokenizer: _FakeTokenizer, fast: bool = False) -> _RecordingCounter: + counter = _RecordingCounter(tokenizer, fast) self.counters.append(counter) return counter - def from_cl100k_ranks(self, rank_file: str) -> _RecordingCounter: - self.rank_files.append(rank_file) - return self("cl100k_base") - - def from_o200k_ranks(self, rank_file: str) -> _RecordingCounter: - self.rank_files.append(rank_file) - return self("o200k_base") - - -class _RaisingCounter: - def __init__(self, error: Exception) -> None: - self.error = error - - async def acount_request(self, body: bytes) -> object: - raise self.error - - -class _RaisingFactory: - """Every counter it builds, for either tokenizer, raises `error` on count.""" - - def __init__(self, error: Exception) -> None: - self.error = error - - def __call__(self, tokenizer_json: str) -> _RaisingCounter: - return _RaisingCounter(self.error) - - def from_cl100k_ranks(self, rank_file: str) -> _RaisingCounter: - return _RaisingCounter(self.error) - - def from_o200k_ranks(self, rank_file: str) -> _RaisingCounter: - return _RaisingCounter(self.error) - @pytest.fixture(autouse=True) -def _reset_bridge(monkeypatch: pytest.MonkeyPatch): - bridge.TOKEN_COUNTER.reset() +def _reset_counters(): bridge._counter.cache_clear() - configuration.reset_rust_configuration() - monkeypatch.setattr(bindings, "get_native_bridge", lambda: _FakeNative()) yield - bridge.TOKEN_COUNTER.reset() bridge._counter.cache_clear() - configuration.reset_rust_configuration() + + +@pytest.fixture +def fake_tokenizers(monkeypatch: pytest.MonkeyPatch) -> None: + """Point the counter's tokenizer lookups at fakes so a recording factory sees which one it was built over.""" + fakes: Final = {name: _FakeTokenizer(name) for name in ("cl100k_base", "o200k_base")} + anthropic: Final = _FakeTokenizer("anthropic", claude_json_str) + monkeypatch.setattr(tokenizer_dispatch, "native_encoding", fakes.__getitem__) + monkeypatch.setattr(tokenizer_dispatch, "native_anthropic", lambda: anthropic) @pytest.mark.asyncio -@pytest.mark.parametrize("tokenizer", TOKENIZERS) -async def test_disabled_bridge_never_constructs_a_counter(tokenizer: bridge.RustTokenizer) -> None: +async def test_native_count_returns_typed_count_and_reuses_one_counter(fake_tokenizers: None) -> None: factory: Final = _RecordingFactory() - litellm.rust(False) - bridge.TOKEN_COUNTER.override(factory) - assert await bridge.count_input_tokens(BODY, tokenizer) is None - assert factory.counters == [] - - -@pytest.mark.asyncio -async def test_enabled_bridge_returns_typed_count_and_reuses_one_counter() -> None: - factory: Final = _RecordingFactory() - litellm.rust(True) - bridge.TOKEN_COUNTER.override(factory) - - first: Final = await bridge.count_input_tokens(BODY, "anthropic") - second: Final = await bridge.count_input_tokens(BODY, "anthropic") + first: Final = await bridge.native_count(factory, "anthropic", BODY) + second: Final = await bridge.native_count(factory, "anthropic", BODY) assert first == bridge.InputTokenCount(model=MODEL, input_tokens=42) assert second == first assert len(factory.counters) == 1 assert factory.counters[0].bodies == [BODY, BODY] - assert json.loads(factory.counters[0].tokenizer_json)["model"]["type"] == "BPE" + assert factory.counters[0].fast is False + assert factory.counters[0].tokenizer is tokenizer_dispatch.native_anthropic() + assert json.loads(factory.counters[0].tokenizer.json or "")["model"]["type"] == "BPE" @pytest.mark.asyncio @pytest.mark.parametrize("tokenizer", ("cl100k_base", "o200k_base")) -async def test_tiktoken_counter_is_built_from_the_vendored_rank_file_once(tokenizer: bridge.RustTokenizer) -> None: +async def test_tiktoken_counter_is_built_over_the_shared_encoding_once( + fake_tokenizers: None, tokenizer: bridge.RustTokenizer +) -> None: factory: Final = _RecordingFactory() - litellm.rust(True) - bridge.TOKEN_COUNTER.override(factory) - first: Final = await bridge.count_input_tokens(BODY, tokenizer) - second: Final = await bridge.count_input_tokens(BODY, tokenizer) + first: Final = await bridge.native_count(factory, tokenizer, BODY) + second: Final = await bridge.native_count(factory, tokenizer, BODY) assert first == second == bridge.InputTokenCount(model=MODEL, input_tokens=42) - assert len(factory.rank_files) == 1 - assert factory.rank_files[0].startswith("IQ== 0\n") - assert factory.rank_files[0].count("\n") == RANK_FILE_LINES[tokenizer] - assert factory.counters[0].tokenizer_json == tokenizer + assert len(factory.counters) == 1 + assert factory.counters[0].tokenizer.name == tokenizer + assert factory.counters[0].tokenizer is tokenizer_dispatch.native_encoding(tokenizer) + assert factory.counters[0].fast is False assert factory.counters[0].bodies == [BODY, BODY] @pytest.mark.asyncio -async def test_each_tokenizer_gets_its_own_cached_counter() -> None: +async def test_each_tokenizer_gets_its_own_cached_counter(fake_tokenizers: None) -> None: factory: Final = _RecordingFactory() - litellm.rust(True) - bridge.TOKEN_COUNTER.override(factory) - await bridge.count_input_tokens(BODY, "anthropic") - await bridge.count_input_tokens(BODY, "cl100k_base") - await bridge.count_input_tokens(BODY, "o200k_base") - await bridge.count_input_tokens(BODY, "anthropic") - await bridge.count_input_tokens(BODY, "o200k_base") + await bridge.native_count(factory, "anthropic", BODY) + await bridge.native_count(factory, "cl100k_base", BODY) + await bridge.native_count(factory, "o200k_base", BODY) + await bridge.native_count(factory, "anthropic", BODY) + await bridge.native_count(factory, "o200k_base", BODY) - assert [counter.tokenizer_json for counter in factory.counters][1:] == ["cl100k_base", "o200k_base"] + assert [counter.tokenizer.name for counter in factory.counters] == ["anthropic", "cl100k_base", "o200k_base"] assert [len(counter.bodies) for counter in factory.counters] == [2, 1, 2] -@pytest.mark.asyncio -async def test_missing_native_module_falls_back(monkeypatch: pytest.MonkeyPatch) -> None: - litellm.rust(True) - monkeypatch.setattr(bindings, "get_native_bridge", lambda: None) - - assert [await bridge.count_input_tokens(BODY, tokenizer) for tokenizer in TOKENIZERS] == [None, None, None] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("tokenizer", TOKENIZERS) -async def test_declined_request_falls_back(tokenizer: bridge.RustTokenizer) -> None: - litellm.rust(True) - bridge.TOKEN_COUNTER.override(_RaisingFactory(_FakeDeclined("request has no messages"))) - - assert await bridge.count_input_tokens(BODY, tokenizer) is None - - -@pytest.mark.asyncio -@pytest.mark.parametrize("tokenizer", TOKENIZERS) -async def test_runtime_failure_falls_back(tokenizer: bridge.RustTokenizer) -> None: - litellm.rust(True) - bridge.TOKEN_COUNTER.override(_RaisingFactory(RuntimeError("encode failed"))) - - assert await bridge.count_input_tokens(BODY, tokenizer) is None - - @pytest.mark.parametrize( ("model", "expected"), ( @@ -215,7 +148,6 @@ async def test_runtime_failure_falls_back(tokenizer: bridge.RustTokenizer) -> No ("gpt-4o", "o200k_base"), ("gpt-4o-mini", "o200k_base"), ("gpt-4o-2024-08-06", "o200k_base"), - ("chatgpt-4o-latest", "o200k_base"), ("gpt-4.1", "o200k_base"), ("gpt-5", "o200k_base"), ("gpt-5-mini", "o200k_base"), @@ -273,8 +205,8 @@ def test_rust_tokenizer_names_the_encoding_python_actually_counts_with(model: st "Hello, world! camelCase ABCdef \u00e9\u00e8 12345 \u3053\u3093\u306b\u3061\u306f <|endoftext|>\r\n" * 9 ) python_count: Final = litellm.token_counter(model=model, text=text) - cl100k_count: Final = len(tiktoken.get_encoding("cl100k_base").encode(text, disallowed_special=())) - o200k_count: Final = len(tiktoken.get_encoding("o200k_base").encode(text, disallowed_special=())) + cl100k_count: Final = Tokenizer.from_tiktoken("cl100k_base").count(text) + o200k_count: Final = Tokenizer.from_tiktoken("o200k_base").count(text) assert cl100k_count != o200k_count match bridge.rust_tokenizer(model): case "cl100k_base": @@ -282,7 +214,7 @@ def test_rust_tokenizer_names_the_encoding_python_actually_counts_with(model: st case "o200k_base": assert python_count == o200k_count case "anthropic": - assert python_count == len(Tokenizer.from_str(claude_json_str).encode(text).ids) + assert python_count == Tokenizer.from_json(claude_json_str).count(text) assert python_count not in {cl100k_count, o200k_count} case None: pytest.fail(f"{model} must have a Rust tokenizer") @@ -379,41 +311,34 @@ PARITY_MODELS: Final[tuple[tuple[str, bridge.RustTokenizer], ...]] = ( @pytest.mark.parametrize(("model", "tokenizer"), PARITY_MODELS) @pytest.mark.parametrize("request_body", PARITY_REQUESTS) async def test_native_count_matches_python_budget_counter( - monkeypatch: pytest.MonkeyPatch, request_body: dict[str, object], model: str, tokenizer: bridge.RustTokenizer + request_body: dict[str, object], model: str, tokenizer: bridge.RustTokenizer ) -> None: native: Final = pytest.importorskip("litellm.rust_bridge._native") - monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) - litellm.rust(True) body: Final = json.dumps(request_body).replace(MODEL, model) + parsed: Final = json.loads(body) - rust_count: Final = await bridge.count_input_tokens(body.encode(), tokenizer) - python_count: Final = _count_input_tokens(request_body=json.loads(body), model=model) + counted: Final = await bridge.native_count(native.TokenCounter, tokenizer, body.encode()) - assert rust_count is not None - assert rust_count.model == json.loads(body).get("model") - assert rust_count.input_tokens == python_count + assert counted.input_tokens == count_input_tokens_for_model(request_body=parsed, model=model) @pytest.mark.asyncio @pytest.mark.parametrize(("model", "tokenizer"), ((CL100K_MODEL, "cl100k_base"), (O200K_MODEL, "o200k_base"))) async def test_tiktoken_counts_long_text_exactly_where_python_chunks( - monkeypatch: pytest.MonkeyPatch, model: str, tokenizer: bridge.RustTokenizer + model: str, tokenizer: bridge.RustTokenizer ) -> None: """Python encodes tiktoken text in fixed-size chunks (drift of up to one token per chunk boundary); Rust does not.""" native: Final = pytest.importorskip("litellm.rust_bridge._native") - monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) - litellm.rust(True) text: Final = "x " * 20_000 body: Final = {"model": model, "messages": [{"role": "user", "content": text}]} - encoding: Final = tiktoken.get_encoding(tokenizer) - exact: Final = 3 + len(encoding.encode("user")) + len(encoding.encode(text)) + 3 + encoding: Final = Tokenizer.from_tiktoken(tokenizer) + exact: Final = 3 + encoding.count("user") + encoding.count(text) + 3 chunks: Final = -(-len(text) // TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS) - rust_count: Final = await bridge.count_input_tokens(json.dumps(body).encode(), tokenizer) - python_count: Final = _count_input_tokens(request_body=body, model=model) + counted: Final = await bridge.native_count(native.TokenCounter, tokenizer, json.dumps(body).encode()) + python_count: Final = count_input_tokens_for_model(request_body=body, model=model) - assert rust_count is not None - assert rust_count.input_tokens == exact + assert counted.input_tokens == exact assert python_count is not None assert exact < python_count <= exact + chunks @@ -435,10 +360,10 @@ DECLINED_REQUESTS: Final[tuple[dict[str, object], ...]] = ( @pytest.mark.parametrize("tokenizer", TOKENIZERS) @pytest.mark.parametrize("request_body", DECLINED_REQUESTS) async def test_native_declines_shapes_python_prices_differently( - monkeypatch: pytest.MonkeyPatch, request_body: dict[str, object], tokenizer: bridge.RustTokenizer + request_body: dict[str, object], tokenizer: bridge.RustTokenizer ) -> None: native: Final = pytest.importorskip("litellm.rust_bridge._native") - monkeypatch.setattr(bindings, "get_native_bridge", lambda: native) - litellm.rust(True) + raw, _ = _counted(request_body, MODEL_BY_TOKENIZER[tokenizer]) - assert await bridge.count_input_tokens(json.dumps(request_body).encode(), tokenizer) is None + with pytest.raises(native.RustBridgeDeclined): + await bridge.native_count(native.TokenCounter, tokenizer, raw) diff --git a/tests/test_litellm/rust_bridge/test_tokenizer.py b/tests/test_litellm/rust_bridge/test_tokenizer.py new file mode 100644 index 00000000000..188aa81093f --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_tokenizer.py @@ -0,0 +1,49 @@ +from typing import Final + +import pytest +import tiktoken +from tokenizers import Tokenizer + +from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEncoding +from litellm.rust_bridge import tokenizer +from litellm.utils import claude_json_str +from tests.test_litellm.litellm_core_utils.test_decode_special_tokens import TOKENIZER_JSON + +TEXTS: Final = ("hello <|endoftext|> world", "café 漢字 🙂", " def f():\n return 1\n", "hello again") + + +@pytest.mark.parametrize("name", ("cl100k_base", "o200k_base")) +@pytest.mark.parametrize("text", TEXTS) +def test_native_encoding_matches_tiktoken(name: str, text: str) -> None: + native: Final = tokenizer.native_encoding(name) + if native is None: + pytest.skip("native extension is not built") + encoding: Final = OpenAIEncoding.wrap(native) + reference: Final = tiktoken.get_encoding(name) + + ids: Final = encoding.encode(text, disallowed_special=()) + assert ids == reference.encode(text, disallowed_special=()) + assert encoding.decode(ids) == reference.decode(ids) + + +@pytest.mark.parametrize("text", TEXTS) +def test_native_anthropic_tokenizer_matches_python(text: str) -> None: + native: Final = tokenizer.native_anthropic() + if native is None: + pytest.skip("native extension is not built") + reference: Final = Tokenizer.from_str(claude_json_str) + + ids: Final = HuggingFaceTokenizer(native).encode(text).ids + assert ids == reference.encode(text).ids + assert HuggingFaceTokenizer(native).decode(ids) == reference.decode(ids) + + +def test_native_custom_tokenizer_matches_python() -> None: + factory: Final = tokenizer.TOKENIZER.load() + if factory is None: + pytest.skip("native extension is not built") + native: Final = HuggingFaceTokenizer(factory.from_json(TOKENIZER_JSON)) + reference: Final = Tokenizer.from_str(TOKENIZER_JSON) + + assert native.encode("Hello World").ids == reference.encode("Hello World").ids + assert native.decode(reference.encode("Hello World").ids) == reference.decode(reference.encode("Hello World").ids) diff --git a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py index e449d4392d8..0eae041f535 100644 --- a/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py +++ b/tests/test_litellm/rust_bridge/test_verify_linux_native_wheel.py @@ -56,10 +56,11 @@ def _write_wheel( metadata_tags: tuple[str, ...] | None = (_EXPECTED_TAG,), dist_info: str = _DIST_INFO, duplicate_wheel: bool = False, + native_bytes: bytes = b"synthetic native extension", ) -> Path: wheel: Final = tmp_path / f"litellm-1.100.0-{filename_tag}.whl" with zipfile.ZipFile(wheel, "w", compression=zipfile.ZIP_DEFLATED) as archive: - archive.writestr(_NATIVE_MEMBER, b"synthetic native extension") + archive.writestr(_NATIVE_MEMBER, native_bytes) archive.writestr( f"{dist_info}/METADATA", "Metadata-Version: 2.1\nName: litellm\nVersion: 1.100.0\n", @@ -195,3 +196,17 @@ def test_rejects_production_module_exposing_panic_hook(tmp_path: Path) -> None: wheel: Final = _write_wheel(tmp_path, filename_tag=_EXPECTED_TAG) assert _run_verifier(wheel, exposes_panic=True) == 1 + + +@pytest.mark.parametrize("embedded", (False, True)) +def test_vocabulary_is_packaged_once(tmp_path: Path, embedded: bool) -> None: + ranks: Final = b"AA== 0\nAQ== 1\nAg== 2\n" + wheel: Final = _write_wheel( + tmp_path, + filename_tag=_EXPECTED_TAG, + native_bytes=b"native engine" + (ranks if embedded else b""), + ) + with zipfile.ZipFile(wheel, "a") as archive: + archive.writestr("litellm/litellm_core_utils/tokenizers/" + "a" * 40, ranks) + + assert _run_verifier(wheel) == (1 if embedded else 0) diff --git a/tests/test_litellm/secret_managers/test_aws_secret_manager_rotation.py b/tests/test_litellm/secret_managers/test_aws_secret_manager_rotation.py index 4a4cec6bf77..af1380f6fd1 100644 --- a/tests/test_litellm/secret_managers/test_aws_secret_manager_rotation.py +++ b/tests/test_litellm/secret_managers/test_aws_secret_manager_rotation.py @@ -1,11 +1,17 @@ -from collections.abc import Mapping +import json +from collections.abc import Iterator, Mapping +from contextlib import contextmanager from dataclasses import dataclass, replace from types import MappingProxyType from typing import Final, TypeAlias +import httpx import pytest +import litellm +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.secret_managers.aws_secret_manager_v2 import AWSSecretsManagerV2 +from litellm.types.llms.custom_http import httpxSpecialProvider OptionalParams: TypeAlias = Mapping[str, object] | None @@ -204,3 +210,173 @@ async def test_rotate_secret_different_names_persists_requested_value_and_delete assert manager.storage.values[new_name] == new_value assert current_name not in manager.storage.values assert manager.storage.values[unrelated_secret_name] == unrelated_value + + +@dataclass(frozen=True, slots=True) +class FakeSecretsManagerState: + live: Mapping[str, str] + scheduled_for_deletion: frozenset[str] = frozenset() + actions: tuple[str, ...] = () + descriptions: Mapping[str, str] = MappingProxyType({}) + failing_actions: frozenset[str] = frozenset() + + +class FakeSecretsManagerService: + def __init__(self, state: FakeSecretsManagerState) -> None: + self.state = state + + def handle(self, request: httpx.Request) -> httpx.Response: + action: Final = request.headers["X-Amz-Target"].removeprefix("secretsmanager.") + body: Final = json.loads(request.content) + name: Final = str(body.get("Name") or body.get("SecretId")) + self.state = replace(self.state, actions=(*self.state.actions, f"{action}:{name}")) + if action in self.state.failing_actions: + return self._error("InternalServiceError", f"injected failure for {action}") + match action: + case "CreateSecret": + if name in self.state.live: + return self._error("ResourceExistsException", f"The secret {name} already exists") + self.state = replace( + self.state, + live=MappingProxyType({**self.state.live, name: str(body["SecretString"])}), + descriptions=MappingProxyType({**self.state.descriptions, name: str(body.get("Description", ""))}), + ) + return httpx.Response(200, json={"ARN": f"arn:fake:{name}", "Name": name}) + case "UpdateSecret": + if name in self.state.scheduled_for_deletion: + return self._error( + "InvalidRequestException", + "You can't perform this operation on the secret because it was marked for deletion.", + ) + self.state = replace( + self.state, + live=MappingProxyType({**self.state.live, name: str(body["SecretString"])}), + descriptions=MappingProxyType({**self.state.descriptions, name: str(body.get("Description", ""))}), + ) + return httpx.Response(200, json={"ARN": f"arn:fake:{name}", "Name": name}) + case "DescribeSecret": + if name not in self.state.live: + return self._error("ResourceNotFoundException", "Secrets Manager can't find the specified secret.") + deleted: Final = "2026-01-01T00:00:00Z" if name in self.state.scheduled_for_deletion else None + return httpx.Response(200, json={"ARN": f"arn:fake:{name}", "Name": name, "DeletedDate": deleted}) + case "RestoreSecret": + self.state = replace(self.state, scheduled_for_deletion=self.state.scheduled_for_deletion - {name}) + return httpx.Response(200, json={"ARN": f"arn:fake:{name}", "Name": name}) + case "PutSecretValue": + if name in self.state.scheduled_for_deletion: + return self._error( + "InvalidRequestException", + "You can't perform this operation on the secret because it was marked for deletion.", + ) + self.state = replace( + self.state, live=MappingProxyType({**self.state.live, name: str(body["SecretString"])}) + ) + return httpx.Response(200, json={"ARN": f"arn:fake:{name}", "Name": name}) + case "GetSecretValue": + if name not in self.state.live or name in self.state.scheduled_for_deletion: + return self._error("ResourceNotFoundException", "Secrets Manager can't find the specified secret.") + return httpx.Response(200, json={"SecretString": self.state.live[name]}) + case "DeleteSecret": + self.state = replace(self.state, scheduled_for_deletion=self.state.scheduled_for_deletion | {name}) + return httpx.Response(200, json={"ARN": f"arn:fake:{name}", "Name": name}) + return self._error("UnsupportedAction", action) + + @staticmethod + def _error(error_type: str, message: str) -> httpx.Response: + return httpx.Response(400, json={"__type": error_type, "message": message}) + + +@contextmanager +def fake_secrets_manager(monkeypatch: pytest.MonkeyPatch) -> Iterator[FakeSecretsManagerService]: + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "synthetic-access-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "synthetic-secret-key") + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + service: Final = FakeSecretsManagerService(FakeSecretsManagerState(live=MappingProxyType({}))) + cache_key: Final = "async_httpx_clienttimeout_None" + httpxSpecialProvider.SecretManager + litellm.in_memory_llm_clients_cache.set_cache( + key=cache_key, + value=AsyncHTTPHandler(transport=httpx.MockTransport(service.handle)), + ) + try: + yield service + finally: + litellm.in_memory_llm_clients_cache.delete_cache( + litellm.in_memory_llm_clients_cache.update_cache_key_with_event_loop(cache_key) + ) + + +@pytest.mark.asyncio +async def test_rotate_secret_back_to_name_inside_recovery_window_restores_and_stores_new_value( + monkeypatch: pytest.MonkeyPatch, +) -> None: + alias_a: Final = "synthetic/alias-a" + alias_b: Final = "synthetic/alias-b" + with fake_secrets_manager(monkeypatch) as fake: + manager: Final = AWSSecretsManagerV2(aws_region_name="us-east-1") + await manager.async_write_secret(secret_name=alias_a, secret_value="value-1") + await manager.async_rotate_secret( + current_secret_name=alias_a, new_secret_name=alias_b, new_secret_value="value-2" + ) + assert alias_a in fake.state.scheduled_for_deletion + + await manager.async_rotate_secret( + current_secret_name=alias_b, new_secret_name=alias_a, new_secret_value="value-3" + ) + + assert await manager.async_read_secret(secret_name=alias_a) == "value-3" + assert await manager.async_read_secret(secret_name=alias_b) is None + assert fake.state.scheduled_for_deletion == frozenset({alias_b}) + assert fake.state.descriptions[alias_a] == f"Rotated from {alias_b}" + + +@pytest.mark.asyncio +async def test_write_secret_to_name_inside_recovery_window_reschedules_deletion_when_update_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + alias: Final = "synthetic/deleted-alias" + with fake_secrets_manager(monkeypatch) as fake: + manager: Final = AWSSecretsManagerV2(aws_region_name="us-east-1") + await manager.async_write_secret(secret_name=alias, secret_value="value-1") + await manager.async_delete_secret(secret_name=alias, recovery_window_in_days=7) + fake.state = replace(fake.state, failing_actions=frozenset({"UpdateSecret"})) + + with pytest.raises(ValueError, match="injected failure for UpdateSecret"): + await manager.async_write_secret(secret_name=alias, secret_value="value-2") + + assert fake.state.scheduled_for_deletion == frozenset({alias}) + assert fake.state.live[alias] == "value-1" + + +@pytest.mark.asyncio +async def test_write_secret_to_name_inside_recovery_window_restores_and_stores_new_value( + monkeypatch: pytest.MonkeyPatch, +) -> None: + alias: Final = "synthetic/deleted-alias" + with fake_secrets_manager(monkeypatch) as fake: + manager: Final = AWSSecretsManagerV2(aws_region_name="us-east-1") + await manager.async_write_secret(secret_name=alias, secret_value="value-1") + await manager.async_delete_secret(secret_name=alias, recovery_window_in_days=7) + + assert await manager.async_write_secret(secret_name=alias, secret_value="value-2") == { + "ARN": f"arn:fake:{alias}", + "Name": alias, + } + + assert await manager.async_read_secret(secret_name=alias) == "value-2" + assert fake.state.scheduled_for_deletion == frozenset() + + +@pytest.mark.asyncio +async def test_write_secret_to_live_existing_name_still_fails_without_overwriting( + monkeypatch: pytest.MonkeyPatch, +) -> None: + alias: Final = "synthetic/live-alias" + with fake_secrets_manager(monkeypatch) as fake: + manager: Final = AWSSecretsManagerV2(aws_region_name="us-east-1") + await manager.async_write_secret(secret_name=alias, secret_value="value-1") + + with pytest.raises(ValueError, match="ResourceExistsException"): + await manager.async_write_secret(secret_name=alias, secret_value="value-2") + + assert await manager.async_read_secret(secret_name=alias) == "value-1" + assert f"RestoreSecret:{alias}" not in fake.state.actions diff --git a/tests/test_litellm/test_anthropic_beta_headers_filtering.py b/tests/test_litellm/test_anthropic_beta_headers_filtering.py index 19b26120672..8656a7564d2 100644 --- a/tests/test_litellm/test_anthropic_beta_headers_filtering.py +++ b/tests/test_litellm/test_anthropic_beta_headers_filtering.py @@ -83,6 +83,7 @@ class TestAnthropicBetaHeadersFiltering: filtered = filter_and_transform_beta_headers( beta_headers=all_headers, provider=provider ) + assert ("compact-2026-09-04" in filtered) is (provider == "anthropic") for header in unsupported_headers: assert ( diff --git a/tests/test_litellm/test_claude_opus_5_config.py b/tests/test_litellm/test_claude_opus_5_config.py index aaf179e0216..b5efe016a60 100644 --- a/tests/test_litellm/test_claude_opus_5_config.py +++ b/tests/test_litellm/test_claude_opus_5_config.py @@ -1,6 +1,8 @@ """ Validate Claude Opus 5 model configuration entries. +Opus 5.5 (``claude-opus-5-5``) is covered here too. + Opus 5 carries Opus 4.8's pricing ($5 / $25 per MTok) and the gen-5 adaptive thinking profile, but differs from 4.8 in two ways that are behavior-bearing in LiteLLM: the cacheable-prefix minimum drops to 512 tokens, and Bedrock's Opus 5 @@ -12,14 +14,23 @@ validator accepts the full effort ladder, so the entries must not carry the ``anthropic/*`` wildcard deployment). """ +import json import os import pytest from litellm.constants import BEDROCK_CONVERSE_MODELS +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap REPO_ROOT = os.path.join(os.path.dirname(__file__), "../..") + +def _load_root_cost_map() -> dict: + json_path = os.path.join(REPO_ROOT, "model_prices_and_context_window.json") + with open(json_path) as f: + return json.load(f) + + ALL_OPUS_5_VARIANTS = ( "claude-opus-5", "anthropic.claude-opus-5", @@ -30,7 +41,10 @@ ALL_OPUS_5_VARIANTS = ( "jp.anthropic.claude-opus-5", "vertex_ai/claude-opus-5", "vertex_ai/claude-opus-5@default", + "vertex_ai/claude-opus-5-5", + "vertex_ai/claude-opus-5-5@default", "azure_ai/claude-opus-5", + "azure_ai/claude-opus-5-5", ) BEDROCK_OPUS_5_VARIANTS = ( @@ -58,3 +72,37 @@ def test_opus_5_registered_for_bedrock_converse(): assert "anthropic.claude-opus-5" in BEDROCK_CONVERSE_MODELS +OPUS_5_5_VARIANTS = ( + "claude-opus-5-5", + "vertex_ai/claude-opus-5-5", + "vertex_ai/claude-opus-5-5@default", + "azure_ai/claude-opus-5-5", +) + + +@pytest.mark.parametrize("model_name", OPUS_5_5_VARIANTS) +def test_opus_5_5_present_in_bundled_backup(model_name): + backup = GetModelCostMap.load_local_model_cost_map() + root = _load_root_cost_map() + assert model_name in backup + assert model_name in root + assert backup[model_name] == root[model_name] + + +@pytest.mark.parametrize( + ("model", "provider"), + [ + ("claude-opus-5-5", "anthropic"), + ("anthropic/claude-opus-5-5", "anthropic"), + ("vertex_ai/claude-opus-5-5", "vertex_ai"), + ("azure_ai/claude-opus-5-5", "azure_ai"), + ], +) +def test_opus_5_5_thinking_profile(local_model_cost_map, model, provider): + """Opus 5.5 has thinking always on with the adaptive thinking surface, and + no forced tool use, same as Fable 5.1.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + assert AnthropicModelInfo._is_adaptive_thinking_model(model, provider) is True + assert AnthropicModelInfo._is_always_on_thinking_model(model, provider) is True + assert AnthropicModelInfo.forced_tool_use_unsupported(model.removeprefix("anthropic/")) is True diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 1d6c229f9ce..e7ce9a74797 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1,5 +1,8 @@ +import datetime import time -from typing import Final +from pathlib import Path +from types import MappingProxyType, SimpleNamespace +from typing import Final, cast import pytest from pydantic import BaseModel @@ -20,8 +23,13 @@ from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import OpenAIRealtimeStreamList, ResponseAPIUsage, ResponsesAPIResponse from litellm.types.rerank import RerankResponse from litellm.types.utils import ( + CacheCreationTokenDetails, CallTypes, Choices, + ImageObject, + ImageResponse, + ImageUsage, + ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, @@ -29,6 +37,7 @@ from litellm.types.utils import ( PromptTokensDetailsWrapper, Usage, ) +from litellm.types.videos.main import VideoObject @pytest.fixture @@ -150,131 +159,8 @@ def test_cost_calculator_with_response_cost_in_additional_headers(): assert result == 1000 -def test_cost_calculator_with_usage(_local_model_cost_map, monkeypatch): - - usage = Usage( - prompt_tokens=120, - completion_tokens=100, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=10, - audio_tokens=90, - image_tokens=20, - ), - ) - mr = ModelResponse(usage=usage, model="gemini-2.0-flash-001") - - result = response_cost_calculator( - response_object=mr, - model="", - custom_llm_provider="vertex_ai", - call_type="acompletion", - optional_params={}, - cache_hit=None, - base_model=None, - ) - - model_info = litellm.model_cost["gemini-2.0-flash-001"] - - # Step 1: Test a model where input_cost_per_image_token is not set. - # In this case the calculation should use input_cost_per_token as fallback. - assert model_info.get("input_cost_per_image_token") is None, ( - "Test case expects that input_cost_per_image_token is not set" - ) - - expected_cost = ( - usage.prompt_tokens_details.audio_tokens * model_info["input_cost_per_audio_token"] - + usage.prompt_tokens_details.text_tokens * model_info["input_cost_per_token"] - + usage.prompt_tokens_details.image_tokens * model_info["input_cost_per_token"] - + usage.completion_tokens * model_info["output_cost_per_token"] - ) - - assert result == expected_cost, f"Got {result}, Expected {expected_cost}" - - # Step 2: Set input_cost_per_image_token. - # In this case the explicit cost information should be used. - temp_model_info_object = dict(model_info) - temp_model_info_object["input_cost_per_image_token"] = 0.5 - - monkeypatch.setattr( - litellm, - "model_cost", - {"gemini-2.0-flash-001": temp_model_info_object}, - ) - - # Invalidate caches after modifying litellm.model_cost - from litellm.utils import _invalidate_model_cost_lowercase_map - - _invalidate_model_cost_lowercase_map() - - result = response_cost_calculator( - response_object=mr, - model="", - custom_llm_provider="vertex_ai", - call_type="acompletion", - optional_params={}, - cache_hit=None, - base_model=None, - ) - - expected_cost = ( - usage.prompt_tokens_details.audio_tokens * temp_model_info_object["input_cost_per_audio_token"] - + usage.prompt_tokens_details.text_tokens * temp_model_info_object["input_cost_per_token"] - + usage.prompt_tokens_details.image_tokens * temp_model_info_object["input_cost_per_image_token"] - + usage.completion_tokens * temp_model_info_object["output_cost_per_token"] - ) - - assert result == expected_cost, f"Got {result}, Expected {expected_cost}" -def test_handle_realtime_stream_cost_calculation_stores_cost_breakdown(): - """Regression: realtime cost must populate logging_obj.cost_breakdown so the - spend logs / UI show input vs output cost (issue: cost_breakdown was None for - /v1/realtime even though a total spend was computed).""" - from datetime import datetime - - from litellm.litellm_core_utils.litellm_logging import Logging - - results: OpenAIRealtimeStreamList = [ - {"type": "session.created", "session": {"model": "gpt-4o-realtime-preview"}}, - { - "type": "response.done", - "response": { - "usage": { - "input_tokens": 100, - "output_tokens": 50, - "total_tokens": 150, - } - }, - }, - ] - combined_usage_object = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results( - results=results, - ) - - logging_obj = Logging( - model="gpt-4o-realtime-preview", - messages=[], - stream=False, - call_type="_arealtime", - start_time=datetime.now(), - litellm_call_id="realtime-cost-breakdown-test", - function_id="realtime-cost-breakdown-test", - ) - - total_cost = handle_realtime_stream_cost_calculation( - results=results, - combined_usage_object=combined_usage_object, - custom_llm_provider="openai", - litellm_model_name="gpt-4o-realtime-preview", - litellm_logging_obj=logging_obj, - ) - - assert total_cost > 0 - assert logging_obj.cost_breakdown is not None - assert logging_obj.cost_breakdown["input_cost"] > 0 - assert logging_obj.cost_breakdown["output_cost"] > 0 - assert abs(logging_obj.cost_breakdown["input_cost"] + logging_obj.cost_breakdown["output_cost"] - total_cost) < 1e-9 - assert abs(logging_obj.cost_breakdown["total_cost"] - total_cost) < 1e-9 def test_realtime_stream_combines_text_and_audio_token_details(): @@ -683,6 +569,90 @@ def test_tiered_pricing_only_deployment_selects_router_model_id(): assert router_model_id in selected +@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) +def test_completion_cost_image_generation_reads_deployment_model_info_price_from_logging_metadata( + _local_model_cost_map: None, metadata_key: str +) -> None: + cost = completion_cost( + completion_response=ImageResponse(data=[ImageObject(url="https://example.com/img.png")]), + model="fal_ai/fal-ai/unlisted-image-model", + call_type="image_generation", + custom_pricing=True, + litellm_logging_obj=SimpleNamespace( + litellm_params={metadata_key: {"model_info": {"output_cost_per_image": 0.08}}} + ), + ) + + assert cost == pytest.approx(0.08) + + +def test_completion_cost_image_generation_registered_deployment_price_keeps_map_token_rates( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + deployment_id: Final = "gemini-image-deployment-priced-per-image" + monkeypatch.setitem( + litellm.model_cost, + deployment_id, + {"mode": "image_generation", "litellm_provider": "gemini", "output_cost_per_image": 0.1}, + ) + usage: Final = ImageUsage( + input_tokens=10, + input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=10), + output_tokens=1290, + total_tokens=1300, + ) + + cost = completion_cost( + completion_response=ImageResponse(data=[ImageObject(url="https://example.com/img.png")], usage=usage), + model="gemini/gemini-3.1-flash-image-preview", + custom_llm_provider="gemini", + call_type="image_generation", + custom_pricing=True, + router_model_id=deployment_id, + litellm_logging_obj=SimpleNamespace(litellm_params={"metadata": {"model_info": {"id": deployment_id}}}), + ) + + assert cost == pytest.approx(10 * 5e-07 + 1290 * 6e-05) + + +def test_completion_cost_image_generation_ignores_deployment_model_info_without_custom_pricing( + _local_model_cost_map: None, +) -> None: + cost = completion_cost( + completion_response=ImageResponse(data=[ImageObject(url="https://example.com/img.png")]), + model="fal_ai/openai/gpt-image-2", + call_type="image_generation", + custom_pricing=False, + optional_params={"quality": "high", "image_size": {"width": 1024, "height": 1024}}, + litellm_logging_obj=SimpleNamespace( + litellm_params={"litellm_metadata": {"model_info": {"output_cost_per_image": 0.5}}} + ), + ) + + assert cost == pytest.approx(0.211) + + +async def test_router_image_generation_bills_litellm_params_output_cost_per_image() -> None: + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "img", + "litellm_params": { + "model": "fal_ai/fal-ai/unlisted-image-model", + "api_key": "sk-fake", + "output_cost_per_image": 0.08, + }, + } + ] + ) + + response = await router.aimage_generation(model="img", prompt="x", mock_response="https://example.com/img.png") + + assert response._hidden_params["response_cost"] == pytest.approx(0.08) + + def test_tiered_pricing_only_deployment_completion_cost_is_nonzero(): """End-to-end: a tier-only deployment must produce the tiered cost, not $0. Mirrors the reported dashscope/qwen3.7-plus trace (12 prompt + 377 @@ -1031,126 +1001,6 @@ def test_bedrock_cost_calculator_comparison_with_without_cache(): print(f"Cost with cache: {cost_with_cache}") -def test_log_context_cost_calculation(): - """ - Test that log context cost calculation works correctly with tiered pricing. - - This test verifies that when using extended context (above 200k tokens), - the log context costs are calculated using the appropriate tiered rates. - """ - from litellm import completion_cost - from litellm.types.utils import ( - Choices, - Message, - ModelResponse, - PromptTokensDetailsWrapper, - Usage, - ) - - # Create a mock response with extended context usage - extended_context_response = ModelResponse( - id="test-extended-context-response", - created=1750733889, - model="claude-4-sonnet-20250514", - object="chat.completion", - system_fingerprint=None, - choices=[ - Choices( - finish_reason="stop", - index=0, - message=Message( - content="This is a test response for extended context cost calculation.", - role="assistant", - tool_calls=None, - function_call=None, - ), - ) - ], - usage=Usage( - total_tokens=350000, # Above 200k threshold - prompt_tokens=301000, # Above 200k threshold - completion_tokens=50000, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=300000, - cached_tokens=0, # No cache hits - audio_tokens=None, - image_tokens=None, - character_count=None, - video_length_seconds=None, - cache_creation_tokens=1000, - ), - completion_tokens_details=None, - _cache_creation_input_tokens=1000, # Some tokens added to cache - ), - ) - - # Calculate the cost using the extended context model - result = completion_cost( - completion_response=extended_context_response, - model="claude-4-sonnet-20250514", - custom_llm_provider="anthropic", - ) - - # Debug: Print the actual result - print(f"DEBUG: Actual cost result: ${result:.6f}") - - # Get model info to understand the pricing - from litellm import get_model_info - - model_info = get_model_info(model="claude-4-sonnet-20250514", custom_llm_provider="anthropic") - - # Calculate expected cost based on actual model pricing - input_cost_per_token = model_info.get("input_cost_per_token", 0) - output_cost_per_token = model_info.get("output_cost_per_token", 0) - cache_creation_cost_per_token = model_info.get("cache_creation_input_token_cost", 0) - - # Check if tiered pricing is applied - input_cost_above_200k = model_info.get("input_cost_per_token_above_200k_tokens", input_cost_per_token) - output_cost_above_200k = model_info.get("output_cost_per_token_above_200k_tokens", output_cost_per_token) - cache_creation_above_200k = model_info.get( - "cache_creation_input_token_cost_above_200k_tokens", - cache_creation_cost_per_token, - ) - - print(f"DEBUG: Base input cost per token: ${input_cost_per_token:.2e}") - print(f"DEBUG: Base output cost per token: ${output_cost_per_token:.2e}") - print(f"DEBUG: Base cache creation cost per token: ${cache_creation_cost_per_token:.2e}") - - # Handle tiered pricing - if not available, use base pricing - if input_cost_above_200k is not None: - print(f"DEBUG: Tiered input cost per token (>200k): ${input_cost_above_200k:.2e}") - else: - print("DEBUG: No tiered input pricing available, using base pricing") - input_cost_above_200k = input_cost_per_token - - if output_cost_above_200k is not None: - print(f"DEBUG: Tiered output cost per token (>200k): ${output_cost_above_200k:.2e}") - else: - print("DEBUG: No tiered output pricing available, using base pricing") - output_cost_above_200k = output_cost_per_token - - if cache_creation_above_200k is not None: - print(f"DEBUG: Tiered cache creation cost per token (>200k): ${cache_creation_above_200k:.2e}") - else: - print("DEBUG: No tiered cache creation pricing available, using base pricing") - cache_creation_above_200k = cache_creation_cost_per_token - - # Since we're above 200k tokens, we should use tiered pricing if available - expected_input_cost = 300000 * input_cost_above_200k - expected_output_cost = 50000 * output_cost_above_200k - expected_cache_cost = 1000 * cache_creation_above_200k - expected_total = expected_input_cost + expected_output_cost + expected_cache_cost - - print(f"DEBUG: Expected total: ${expected_total:.6f}") - - # Allow for small floating point differences - assert abs(result - expected_total) < 1e-6, f"Expected cost ${expected_total:.6f}, but got ${result:.6f}" - - print(f"✓ Log context cost calculation with tiered pricing is correct: ${result:.6f}") - print(f" - Input tokens (300k): ${expected_input_cost:.6f}") - print(f" - Output tokens (50k): ${expected_output_cost:.6f}") - print(f" - Cache creation (1k): ${expected_cache_cost:.6f}") - print(f" - Total: ${result:.6f}") def test_gemini_25_explicit_caching_cost_direct_usage(): @@ -1721,56 +1571,6 @@ def test_cost_margin_with_discount(monkeypatch): print(f" - Expected: ${expected_cost:.6f}") -def test_azure_image_generation_cost_calculator(): - from unittest.mock import MagicMock - - from litellm.types.utils import ( - ImageObject, - ImageResponse, - ImageUsage, - ImageUsageInputTokensDetails, - ) - - response_cost_calculator_kwargs = { - "response_object": ImageResponse( - created=1761785270, - background=None, - data=[ - ImageObject( - b64_json=None, - revised_prompt="A futuristic, techno-inspired green duck wearing cool modern sunglasses. The duck has a sleek, metallic appearance with glowing neon green accents, standing on a high-tech urban background with holographic billboards and illuminated city lights in the distance. The duck's feathers have a glossy, high-tech sheen, resembling a robotic design but still maintaining its avian features. The scene has a vibrant, cyberpunk aesthetic with a neon color palette.", - url="test-azure-blob-url-with-sas-token", - ) - ], - output_format=None, - quality="hd", - size=None, - usage=ImageUsage( - input_tokens=0, - input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=0), - output_tokens=0, - total_tokens=0, - ), - ), - "model": "azure/dall-e-3", - "cache_hit": False, - "custom_llm_provider": "azure", - "base_model": "azure/dall-e-3", - "call_type": "aimage_generation", - "optional_params": {}, - "custom_pricing": False, - "prompt": "", - "standard_built_in_tools_params": { - "web_search_options": None, - "file_search": None, - }, - "router_model_id": "6738c432ffc9b733597c6b86613ca20dc5f49bde591fd3d03e7cd6aa25bb241e", - "litellm_logging_obj": MagicMock(), - "service_tier": None, - } - - cost = response_cost_calculator(**response_cost_calculator_kwargs) - assert cost > 0.079 def test_completion_cost_extracts_service_tier_from_response(_local_model_cost_map): @@ -2523,87 +2323,6 @@ def test_gemini_without_cache_tokens_details(): print("✅ Gemini without cacheTokensDetails works correctly") -def test_gemini_implicit_caching_cost_calculation(): - """ - Test for Issue #16341: Gemini implicit cached tokens not counted in spend log - - When Gemini uses implicit caching, it returns cachedContentTokenCount but NOT - cacheTokensDetails. In this case, we should subtract cachedContentTokenCount - from text_tokens to correctly calculate costs. - - See: https://github.com/BerriAI/litellm/issues/16341 - """ - from litellm import completion_cost - from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig, - ) - from litellm.types.utils import Choices, Message, ModelResponse - - # Simulate Gemini response with implicit caching (cachedContentTokenCount only) - completion_response = { - "usageMetadata": { - "promptTokenCount": 10000, - "candidatesTokenCount": 5, - "totalTokenCount": 10005, - "cachedContentTokenCount": 8000, # Implicit caching - no cacheTokensDetails - "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 10000}], - "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 5}], - } - } - - usage = VertexGeminiConfig._calculate_usage(completion_response) - - # Verify parsing - assert usage.cache_read_input_tokens == 8000, ( - f"cache_read_input_tokens should be 8000, got {usage.cache_read_input_tokens}" - ) - assert usage.prompt_tokens_details.cached_tokens == 8000, ( - f"cached_tokens should be 8000, got {usage.prompt_tokens_details.cached_tokens}" - ) - - # CRITICAL: text_tokens should be (10000 - 8000) = 2000, NOT 10000 - # This is the fix for issue #16341 - assert usage.prompt_tokens_details.text_tokens == 2000, ( - f"text_tokens should be 2000 (10000 - 8000), got {usage.prompt_tokens_details.text_tokens}" - ) - - # Verify cost calculation uses cached token pricing - response = ModelResponse( - id="mock-id", - model="gemini-2.0-flash", - choices=[ - Choices( - index=0, - message=Message(role="assistant", content="Hello!"), - finish_reason="stop", - ) - ], - usage=usage, - ) - - cost = completion_cost( - completion_response=response, - model="gemini-2.0-flash", - custom_llm_provider="gemini", - ) - - # Get model pricing for verification - import litellm - - model_info = litellm.get_model_info("gemini/gemini-2.0-flash") - input_cost = model_info.get("input_cost_per_token", 0) - cache_read_cost = model_info.get("cache_read_input_token_cost", input_cost) - output_cost = model_info.get("output_cost_per_token", 0) - - # Expected cost: (2000 * input) + (8000 * cache_read) + (5 * output) - expected_cost = (2000 * input_cost) + (8000 * cache_read_cost) + (5 * output_cost) - - assert abs(cost - expected_cost) < 1e-9, ( - f"Cost calculation is wrong. Got ${cost:.6f}, expected ${expected_cost:.6f}. " - f"Cached tokens may not be using reduced pricing." - ) - - print("✅ Issue #16341 fix verified: Gemini implicit caching cost calculated correctly") def test_additional_costs_only_for_azure_ai(_local_model_cost_map): @@ -3038,21 +2757,23 @@ def test_completion_cost_logs_cache_and_reasoning_breakdown_for_custom_pricing() assert total == pytest.approx(100 * 1e-6 + 800 * 1e-7 + 100 * 1.25e-6 + 500 * 2e-6) -def test_cost_per_token_per_second_pricing(monkeypatch): +@pytest.mark.parametrize("custom_llm_provider", ["together_ai", "openai", "anthropic", "bedrock", "azure"]) +def test_cost_per_token_per_second_pricing(monkeypatch, custom_llm_provider: str): """ Models priced by duration (input/output_cost_per_second) with no per-token rates - must be billed as cost_per_second * response_time_ms / 1000 in cost_per_token. + must be billed as cost_per_second * response_time_ms / 1000 in cost_per_token, + whether or not the provider has its own cost calculator. """ monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - model = "test-per-second-pricing-model" + model = f"test-per-second-pricing-{custom_llm_provider}" litellm.register_model( model_cost={ model: { "input_cost_per_second": 0.02, "output_cost_per_second": 0.04, - "litellm_provider": "together_ai", + "litellm_provider": custom_llm_provider, "mode": "chat", } } @@ -3060,7 +2781,7 @@ def test_cost_per_token_per_second_pricing(monkeypatch): prompt_cost, completion_cost_value = cost_per_token( model=model, - custom_llm_provider="together_ai", + custom_llm_provider=custom_llm_provider, prompt_tokens=10, completion_tokens=20, response_time_ms=1500.0, @@ -3070,6 +2791,143 @@ def test_cost_per_token_per_second_pricing(monkeypatch): assert completion_cost_value == pytest.approx(0.04 * 1.5) +def test_cost_per_token_keeps_token_pricing_when_per_second_rates_are_also_set(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + model = "test-token-and-per-second-pricing-model" + litellm.register_model( + model_cost={ + model: { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_second": 0.02, + "output_cost_per_second": 0.04, + "litellm_provider": "openai", + "mode": "chat", + } + } + ) + + prompt_cost, completion_cost_value = cost_per_token( + model=model, + custom_llm_provider="openai", + prompt_tokens=10, + completion_tokens=20, + response_time_ms=1500.0, + ) + + assert prompt_cost == pytest.approx(10 * 1e-6) + assert completion_cost_value == pytest.approx(20 * 2e-6) + + +def _logging_obj_with_call_window(duration_ms: float) -> Logging: + start_time: Final = datetime.datetime(2026, 9, 21, 12, 0, 0) + logging_obj: Final = Logging( + model="gpt-5.4-nano", + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=start_time, + litellm_call_id="per-second-call-window", + function_id="f", + ) + logging_obj.model_call_details["start_time"] = start_time + logging_obj.model_call_details["end_time"] = start_time + datetime.timedelta(milliseconds=duration_ms) + return logging_obj + + +@pytest.mark.parametrize( + ("stamped_response_ms", "total_time", "logged_duration_ms", "expected_seconds"), + [(None, 0.0, 1500.0, 1.5), (3000.0, 0.0, 1500.0, 3.0), (None, 2500.0, 1500.0, 2.5), (3000.0, 2500.0, 1500.0, 3.0)], +) +def test_completion_cost_per_second_deployment_bills_the_call_duration( + monkeypatch, + stamped_response_ms: float | None, + total_time: float, + logged_duration_ms: float, + expected_seconds: float, +): + """ + A deployment priced only per second bills the stamped ``_response_ms`` when there is one, + then the caller's explicit ``total_time``, and the logging object's start/end window otherwise + (a streamed response is never stamped). + """ + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + deployment_id = "per-second-openai-deployment" + litellm.register_model( + model_cost={ + deployment_id: { + "input_cost_per_second": 0.02, + "output_cost_per_second": 0.04, + "litellm_provider": "openai", + "mode": "chat", + } + } + ) + response = ModelResponse( + model="gpt-5.4-nano", + usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18), + ) + response._response_ms = stamped_response_ms + + cost = completion_cost( + completion_response=response, + model="openai/gpt-5.4-nano", + custom_llm_provider="openai", + custom_pricing=True, + router_model_id=deployment_id, + total_time=total_time, + litellm_logging_obj=_logging_obj_with_call_window(logged_duration_ms), + ) + + assert cost == pytest.approx((0.02 + 0.04) * expected_seconds) + + +@pytest.mark.parametrize("mode", ["audio_transcription", "audio_speech", "video_generation", "realtime"]) +def test_cost_per_token_leaves_media_second_rates_to_their_dedicated_paths(monkeypatch, mode: str): + """ + A media-mode entry's per-second rates price audio or video seconds, which the dedicated + transcription, speech, video, and realtime paths bill from the media itself, so a call that + reaches the generic path with only a wall-clock duration must not bill them. + """ + model = f"test-media-per-second-{mode}" + monkeypatch.setitem( + litellm.model_cost, + model, + {"input_cost_per_second": 0.02, "output_cost_per_second": 0.4, "litellm_provider": "openai", "mode": mode}, + ) + + assert cost_per_token(model=model, custom_llm_provider="openai", response_time_ms=2000.0) == (0.0, 0.0) + + +def test_completion_cost_video_status_poll_bills_nothing_on_a_per_second_video_model(monkeypatch): + """ + Polling a video job returns a ``VideoObject`` with no stamped duration, so the cost path falls + back to the logging object's call window; on a video model priced per output second that + window must not be billed, or every status poll would charge for the seconds it took to answer. + """ + model = "test-veo-per-second-poll" + monkeypatch.setitem( + litellm.model_cost, + model, + {"output_cost_per_second": 0.4, "litellm_provider": "vertex_ai", "mode": "video_generation"}, + ) + video = VideoObject(id="video_1", object="video", status="completed", model=model, progress=100) + + cost = completion_cost( + completion_response=video, + model=model, + custom_llm_provider="vertex_ai", + call_type=CallTypes.video_retrieve.value, + litellm_logging_obj=_logging_obj_with_call_window(2000.0), + ) + + assert cost == 0.0 + + def _batch_cache_usage() -> Usage: return Usage( prompt_tokens=11000, @@ -3574,6 +3432,45 @@ def test_completion_cost_mantle_native_messages_prices_haiku_from_the_mantle_row ) == pytest.approx(expected), model +def test_completion_cost_legacy_mantle_route_prices_after_router_registration(local_model_cost_map): + """The proxy registers every deployment under its provider-prefixed key at boot. A + bedrock/mantle/ deployment must resolve to the bare Bedrock row there, otherwise the boot + entry is a cost-less capability rule that shadows the priced row and every call on the deployment, + /v1/chat/completions and /v1/messages alike, bills $0.""" + from litellm import Router + + Router( + model_list=[ + { + "model_name": "claude-sonnet-5", + "litellm_params": { + "model": "bedrock/mantle/anthropic.claude-sonnet-5", + "aws_region_name": "us-east-1", + }, + } + ] + ) + assert "bedrock/mantle/anthropic.claude-sonnet-5" not in litellm.model_cost + + response = litellm.ModelResponse( + id="msg_x", + choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + model="claude-sonnet-5", + usage={"prompt_tokens": 16, "completion_tokens": 4, "total_tokens": 20}, + ) + row = litellm.model_cost["anthropic.claude-sonnet-5"] + expected = 16 * row["input_cost_per_token"] + 4 * row["output_cost_per_token"] + assert expected > 0 + + for call_type in ("completion", "anthropic_messages"): + assert litellm.completion_cost( + completion_response=response, + model="mantle/anthropic.claude-sonnet-5", + custom_llm_provider="bedrock", + call_type=call_type, + ) == pytest.approx(expected), call_type + + def test_select_model_name_keeps_base_model_free_of_region(_local_model_cost_map): """An explicit base_model keeps pricing on that model's own key even when the request carries a region with different regional rates, so the private provider model never widens region pricing.""" @@ -4284,6 +4181,273 @@ def test_completion_cost_prices_responses_websocket_turns_per_service_tier(): assert ws_cost != pytest.approx(_http_cost(160, 50, "priority")) +_TIERED_BATCH_MODEL: Final = "lit-tiered-batch-model" +_FLAT_CACHE_BATCH_MODEL: Final = "lit-tiered-batch-model-without-cache-batch-rates" +_TIERED_BATCH_ENTRY: Final = MappingProxyType( + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 2e-6, + "output_cost_per_token": 8e-6, + "cache_read_input_token_cost": 2e-7, + "cache_creation_input_token_cost": 2.5e-6, + "input_cost_per_token_above_272k_tokens": 4e-6, + "output_cost_per_token_above_272k_tokens": 1.2e-5, + "input_cost_per_token_batches": 1e-6, + "output_cost_per_token_batches": 4e-6, + "cache_read_input_token_cost_batches": 1e-7, + "cache_creation_input_token_cost_batches": 1.25e-6, + "input_cost_per_token_above_272k_tokens_batches": 3e-6, + "output_cost_per_token_above_272k_tokens_batches": 7e-6, + "cache_read_input_token_cost_above_272k_tokens_batches": 3e-7, + "cache_creation_input_token_cost_above_272k_tokens_batches": 3.75e-6, + } +) +_BATCH_RATE_PREFIXES: Final = ( + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", +) + + +@pytest.fixture +def _tiered_batch_models(_local_model_cost_map: None) -> None: + litellm.register_model( + model_cost={ + _TIERED_BATCH_MODEL: {**_TIERED_BATCH_ENTRY}, + _FLAT_CACHE_BATCH_MODEL: { + key: rate + for key, rate in _TIERED_BATCH_ENTRY.items() + if not (key.startswith("cache_") and key.endswith("_batches")) + }, + }, + persist_across_reloads=False, + ) + + +def test_batch_cost_calculator_bills_the_long_context_batch_tier_above_272k(_tiered_batch_models: None) -> None: + from litellm.cost_calculator import batch_cost_calculator + + usage: Final = Usage(prompt_tokens=300_035, completion_tokens=64, total_tokens=300_099) + + prompt_cost, completion_cost_value = batch_cost_calculator( + usage=usage, model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(300_035 * 3e-6) + assert completion_cost_value == pytest.approx(64 * 7e-6) + + +@pytest.mark.parametrize("prompt_tokens", [272_000, 1_000]) +def test_batch_cost_calculator_bills_the_flat_batch_rate_at_or_below_272k( + _tiered_batch_models: None, prompt_tokens: int +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + usage: Final = Usage(prompt_tokens=prompt_tokens, completion_tokens=64, total_tokens=prompt_tokens + 64) + + prompt_cost, completion_cost_value = batch_cost_calculator( + usage=usage, model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(prompt_tokens * 1e-6) + assert completion_cost_value == pytest.approx(64 * 4e-6) + + +def test_get_model_info_exposes_every_registered_batch_rate(_tiered_batch_models: None) -> None: + info: Final = litellm.get_model_info(_TIERED_BATCH_MODEL, custom_llm_provider="openai") + batch_keys: Final = tuple(key for key in _TIERED_BATCH_ENTRY if key.endswith("_batches")) + + assert len(batch_keys) == 8 + assert {key: info[key] for key in batch_keys} == {key: _TIERED_BATCH_ENTRY[key] for key in batch_keys} + + +def test_regular_path_never_bills_the_batch_tier_keys(_local_model_cost_map, monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "lit-batch-tier-guard", + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 2e-6, + "output_cost_per_token": 8e-6, + "input_cost_per_token_batches": 1e-6, + "output_cost_per_token_batches": 4e-6, + "input_cost_per_token_above_272k_tokens_batches": 5e-6, + "output_cost_per_token_above_272k_tokens_batches": 9e-6, + }, + ) + + prompt_cost, completion_cost = litellm.cost_per_token( + model="lit-batch-tier-guard", custom_llm_provider="openai", prompt_tokens=300_035, completion_tokens=64 + ) + + assert prompt_cost == pytest.approx(300_035 * 2e-6) + assert completion_cost == pytest.approx(64 * 8e-6) + + +@pytest.mark.parametrize("prefix", _BATCH_RATE_PREFIXES) +def test_every_openai_entry_with_a_long_context_rate_and_a_batch_rate_declares_the_batch_tier( + _local_model_cost_map: None, prefix: str +) -> None: + undeclared: Final = [ + name + for name, entry in litellm.model_cost.items() + if isinstance(entry, dict) + and entry.get("litellm_provider") == "openai" + and entry.get(f"{prefix}_above_272k_tokens") is not None + and entry.get(f"{prefix}_batches") is not None + and entry.get(f"{prefix}_above_272k_tokens_batches") is None + ] + + assert undeclared == [] + + +def test_batch_cost_calculator_ignores_malformed_batch_tier_keys(): + from litellm.cost_calculator import batch_cost_calculator + + usage = Usage(prompt_tokens=300_035, completion_tokens=64, total_tokens=300_099) + model_info = cast( + ModelInfo, + { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 8e-6, + "input_cost_per_token_batches": 1e-6, + "output_cost_per_token_batches": 4e-6, + "input_cost_per_token_above_272k_tokens_batches": 2e-6, + "output_cost_per_token_above_272k_tokens_batches": 6e-6, + "input_cost_per_token_above_lots_tokens_batches": 1.0, + }, + ) + + prompt_cost, completion_cost = batch_cost_calculator( + usage=usage, model=_TIERED_BATCH_MODEL, custom_llm_provider="openai", model_info=model_info + ) + + assert prompt_cost == pytest.approx(300_035 * 2e-6) + assert completion_cost == pytest.approx(64 * 6e-6) + + +def _cached_usage(prompt_tokens: int, cached_tokens: int, completion_tokens: int) -> Usage: + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), + ) + + +def test_batch_cost_calculator_bills_cached_tokens_at_the_long_context_batch_cached_rate( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, completion_cost_value = batch_cost_calculator( + usage=_cached_usage(300_048, 300_045, 11), model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(3 * 3e-6 + 300_045 * 3e-7) + assert completion_cost_value == pytest.approx(11 * 7e-6) + + +def test_batch_cost_calculator_bills_cached_tokens_at_the_flat_batch_cached_rate_at_or_below_272k( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, _ = batch_cost_calculator( + usage=_cached_usage(1_000, 900, 4), model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(100 * 1e-6 + 900 * 1e-7) + + +def test_batch_cost_calculator_bills_cached_tokens_at_the_batch_input_rate_without_a_cached_batch_rate( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, _ = batch_cost_calculator( + usage=_cached_usage(300_048, 300_045, 11), model=_FLAT_CACHE_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(300_048 * 3e-6) + + +def _cache_write_usage(prompt_tokens: int, cache_write_tokens: int, completion_tokens: int) -> Usage: + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=0, cache_write_tokens=cache_write_tokens), + ) + + +def test_batch_cost_calculator_bills_cache_write_tokens_at_the_long_context_batch_cache_write_rate( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, completion_cost_value = batch_cost_calculator( + usage=_cache_write_usage(300_048, 300_045, 4), model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(3 * 3e-6 + 300_045 * 3.75e-6) + assert completion_cost_value == pytest.approx(4 * 7e-6) + + +def test_batch_cost_calculator_bills_cache_write_tokens_at_the_flat_batch_cache_write_rate_at_or_below_272k( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, _ = batch_cost_calculator( + usage=_cache_write_usage(1_000, 900, 4), model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(100 * 1e-6 + 900 * 1.25e-6) + + +def test_batch_cost_calculator_bills_cache_write_tokens_at_the_batch_input_rate_without_a_cache_write_batch_rate( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, _ = batch_cost_calculator( + usage=_cache_write_usage(300_048, 300_045, 4), model=_FLAT_CACHE_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(300_048 * 3e-6) + + +def test_batch_cost_calculator_prices_modalities_and_cached_tokens_together_in_the_crossed_tier() -> None: + from litellm.cost_calculator import batch_cost_calculator + + model_info: Final = cast( + ModelInfo, + { + "input_cost_per_token_batches": 1e-6, + "input_cost_per_token_above_272k_tokens_batches": 3e-6, + "input_cost_per_audio_token_batches": 5e-6, + "cache_read_input_token_cost_batches": 1e-7, + "cache_read_input_token_cost_above_272k_tokens_batches": 3e-7, + }, + ) + usage: Final = Usage( + prompt_tokens=300_000, + completion_tokens=0, + total_tokens=300_000, + prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=64, image_tokens=10, cached_tokens=1_000), + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, model=_TIERED_BATCH_MODEL, custom_llm_provider="openai", model_info=model_info + ) + + assert prompt_cost == pytest.approx(298_926 * 3e-6 + 64 * 5e-6 + 10 * 3e-6 + 1_000 * 3e-7) + + QWEN3_NEXT_REGIONS: Final = ("ap-northeast-1", "ap-south-1", "ap-southeast-2", "eu-west-1", "eu-west-2", "sa-east-1") @@ -4309,3 +4473,27 @@ def test_cost_per_token_bedrock_qwen3_next_uses_regional_entry_not_us_rate( assert prompt_usd == pytest.approx(prompt_tokens * regional["input_cost_per_token"]) assert completion_usd == pytest.approx(completion_tokens * regional["output_cost_per_token"]) + + +def test_cost_per_token_bedrock_nemotron_super_3_uses_eu_west_2_entry_not_us_rate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + regional_key: Final = "bedrock/eu-west-2/nvidia.nemotron-super-3-120b" + regional: Final = litellm.model_cost[regional_key] + us: Final = litellm.model_cost["nvidia.nemotron-super-3-120b"] + assert regional["input_cost_per_token"] != us["input_cost_per_token"] + assert regional["output_cost_per_token"] != us["output_cost_per_token"] + + prompt_tokens, completion_tokens = 1000, 500 + prompt_usd, completion_usd = cost_per_token( + model=regional_key, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + custom_llm_provider="bedrock", + ) + + assert prompt_usd == pytest.approx(prompt_tokens * regional["input_cost_per_token"]) + assert completion_usd == pytest.approx(completion_tokens * regional["output_cost_per_token"]) diff --git a/tests/test_litellm/test_cost_map_guard.py b/tests/test_litellm/test_cost_map_guard.py index 1b4330ed62c..1595bd9864f 100644 --- a/tests/test_litellm/test_cost_map_guard.py +++ b/tests/test_litellm/test_cost_map_guard.py @@ -114,6 +114,40 @@ def test_unclassified_entry_key_is_reported() -> None: assert "Unclassified keys" in failure and "weird_thing" in failure +STALE_HEAD: Final = _snapshot(BASE_MAP, backup=_serialize({**BASE_MAP, "openrouter/b": _entry(3e-06)}), schema="{}") +CODE_ONLY: Final = ( + "litellm/utils.py", + "tests/test_litellm/test_utils.py", + "docs/model_prices_and_context_window.json", +) + + +def test_human_pr_that_leaves_the_cost_map_alone_skips_the_file_checks() -> None: + unparseable: Final = guard.Snapshot(cost_map="{not json", backup="", schema="") + assert _failures(STALE_HEAD, changed_files=CODE_ONLY, bot=False) == () + assert _failures(STALE_HEAD, changed_files=(), bot=False) == () + assert _failures(unparseable, changed_files=CODE_ONLY, bot=False) == () + + +@pytest.mark.parametrize("guarded_path", guard.GUARDED_PATHS) +def test_touching_any_cost_map_file_keeps_the_file_checks(guarded_path: str) -> None: + failures: Final = _failures(STALE_HEAD, changed_files=(*CODE_ONLY, guarded_path), bot=False) + assert [failure for failure in failures if failure.startswith(guard.BACKUP_PATH)] + assert [failure for failure in failures if failure.startswith(guard.SCHEMA_PATH)] + + +def test_bot_pr_always_gets_the_file_checks() -> None: + failures: Final = _failures(STALE_HEAD, changed_files=CODE_ONLY, bot=True) + assert [failure for failure in failures if failure.startswith(guard.BACKUP_PATH)] + assert [failure for failure in failures if failure.startswith(guard.SCHEMA_PATH)] + + +def test_contract_names_the_skip() -> None: + assert guard.contract_for(False, CODE_ONLY) == "human PR, cost map untouched" + assert guard.contract_for(False, (*CODE_ONLY, guard.SCHEMA_PATH)) == "human PR, file checks only" + assert guard.contract_for(True, CODE_ONLY) == "bot contract enforced" + + def test_bot_may_only_touch_the_cost_map_files() -> None: changed = (*guard.GUARDED_PATHS, "litellm/utils.py", ".github/workflows/cost-map-guard.yml") assert _failures(BASE, changed_files=changed, bot=False) == () @@ -147,6 +181,16 @@ def _commit(repo: Path, cost_map: dict[str, object], message: str) -> str: (repo / guard.BACKUP_PATH).parent.mkdir(exist_ok=True) (repo / guard.BACKUP_PATH).write_text(text) (repo / guard.SCHEMA_PATH).write_text(schema_module.render(schema_module.build_schema(cost_map))) + return _git_commit(repo, message) + + +def _commit_code_only(repo: Path, message: str) -> str: + (repo / "litellm").mkdir(exist_ok=True) + (repo / "litellm" / "utils.py").write_text(f"print('{message}')\n") + return _git_commit(repo, message) + + +def _git_commit(repo: Path, message: str) -> str: subprocess.run(("git", "add", "-A"), cwd=repo, check=True) subprocess.run( ("git", "-c", "user.name=t", "-c", "user.email=t@example.com", "commit", "-q", "-m", message), @@ -186,6 +230,40 @@ def test_main_reads_both_revisions_from_git( assert expected_line in result.stdout.splitlines() +def test_main_skips_the_file_checks_on_a_stale_base_the_pr_never_touched(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + _commit(tmp_path, BASE_MAP, "base") + (tmp_path / guard.BACKUP_PATH).write_text(_serialize({**BASE_MAP, "openrouter/b": _entry(3e-06)})) + (tmp_path / guard.SCHEMA_PATH).write_text("{}") + stale_base: Final = _commit_code_only(tmp_path, "stale base with drifted backup and schema") + head: Final = _commit_code_only(tmp_path, "code change on the stale base") + human: Final = _run_guard(tmp_path, stale_base, head, "litellm_fix_pricing") + assert human.returncode == 0, human.stdout + human.stderr + assert "cost map guard passed (human PR, cost map untouched)" in human.stdout.splitlines() + bot: Final = _run_guard(tmp_path, stale_base, head, BOT_REF) + assert bot.returncode == 1 + backup_failure: Final = f"- {guard.BACKUP_PATH} differs from {guard.COST_MAP_PATH}; copy the root file over it" + assert backup_failure in bot.stdout.splitlines() + + +def test_main_keeps_the_file_checks_when_a_cost_map_file_is_renamed(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + base: Final = _commit(tmp_path, BASE_MAP, "base") + subprocess.run(("git", "mv", guard.COST_MAP_PATH, "renamed.json"), cwd=tmp_path, check=True) + head: Final = _git_commit(tmp_path, "rename the cost map") + result: Final = _run_guard(tmp_path, base, head, "litellm_fix_pricing") + assert result.returncode == 1, result.stdout + result.stderr + assert "cost map guard failed (human PR, file checks only):" in result.stdout.splitlines() + + +def test_main_fails_when_the_changed_files_cannot_be_read(tmp_path: Path) -> None: + subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) + head: Final = _commit(tmp_path, BASE_MAP, "head") + result: Final = _run_guard(tmp_path, "0" * 40, head, "litellm_fix_pricing") + assert result.returncode == 1, result.stdout + result.stderr + assert result.stdout.startswith("cost map guard failed: git diff ") + + def test_main_rejects_a_bot_pr_that_edits_code(tmp_path: Path) -> None: subprocess.run(("git", "init", "-q", str(tmp_path)), check=True) base = _commit(tmp_path, BASE_MAP, "base") diff --git a/tests/test_litellm/test_count_tokens_public_api.py b/tests/test_litellm/test_count_tokens_public_api.py index 2918d0aa522..84121dd63e4 100644 --- a/tests/test_litellm/test_count_tokens_public_api.py +++ b/tests/test_litellm/test_count_tokens_public_api.py @@ -157,21 +157,3 @@ def test_acount_tokens_no_api_key_falls_back(monkeypatch): assert result.tokenizer_type == "local_tokenizer" -async def test_acount_tokens_local_fallback_counts_off_the_event_loop(): - from tests.large_text import text - from tests.test_litellm.litellm_core_utils.event_loop_lag import ( - assert_loop_stayed_free, - timed_with_loop_lags, - warm_tokenizer, - ) - - model = "together_ai/meta-llama/Llama-3-8b-chat-hf" - warm_tokenizer(model) - - result, took, lags = await timed_with_loop_lags( - lambda: litellm.acount_tokens(model=model, messages=[{"role": "user", "content": text * 100}]) - ) - - assert result.tokenizer_type == "local_tokenizer" - assert result.total_tokens > 100_000 - assert_loop_stayed_free(took, lags) diff --git a/tests/test_litellm/test_gpt_image_cost_calculator.py b/tests/test_litellm/test_gpt_image_cost_calculator.py index 42d4c699200..d026285f9ae 100644 --- a/tests/test_litellm/test_gpt_image_cost_calculator.py +++ b/tests/test_litellm/test_gpt_image_cost_calculator.py @@ -16,6 +16,8 @@ import litellm from litellm.types.utils import ( ImageObject, ImageResponse, + ImageUsage, + ImageUsageInputTokensDetails, ) @@ -52,31 +54,42 @@ class TestGPTImageCostCalculator: assert cost == 0.0 + @pytest.mark.parametrize( + "usage", + [ + None, + ImageUsage( + input_tokens=0, + input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=0), + output_tokens=0, + total_tokens=0, + ), + ], + ) + def test_gpt_image_1_bills_deployment_output_cost_per_image_without_usage_tokens( + self, usage: ImageUsage | None + ) -> None: + from litellm.llms.openai.image_generation.cost_calculator import cost_calculator + + image_response = ImageResponse( + created=1234567890, + data=[ImageObject(url="http://example.com/one.jpg"), ImageObject(url="http://example.com/two.jpg")], + usage=usage, + ) + + cost = cost_calculator( + model="gpt-image-1", + image_response=image_response, + custom_llm_provider="openai", + model_info={"output_cost_per_image": 0.05}, + ) + + assert cost == pytest.approx(0.10) + class TestGPTImageCostRouting: """Test that gpt-image models are properly routed to the token-based calculator""" - def test_openai_dalle_routes_to_pixel_calculator(self): - """Test that OpenAI DALL-E still routes to pixel-based calculator""" - from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils - - image_response = ImageResponse( - created=1234567890, - data=[ImageObject(url="http://example.com/image.jpg")], - ) - image_response.size = "1024x1024" - image_response.quality = "standard" - - cost = CostCalculatorUtils.route_image_generation_cost_calculator( - model="dall-e-3", - completion_response=image_response, - custom_llm_provider="openai", - size="1024x1024", - quality="standard", - n=1, - ) - - assert cost >= 0 class TestGPTImage15OutputImageTokens: diff --git a/tests/test_litellm/test_openai_service_tier_long_context_pricing.py b/tests/test_litellm/test_openai_service_tier_long_context_pricing.py index c766370230c..9b3a1e57169 100644 --- a/tests/test_litellm/test_openai_service_tier_long_context_pricing.py +++ b/tests/test_litellm/test_openai_service_tier_long_context_pricing.py @@ -58,6 +58,18 @@ PRIORITY_LONG_CONTEXT = { "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, }, + "gpt-6-sol": { + "input_cost_per_token_above_272k_tokens_priority": 8e-06, + "output_cost_per_token_above_272k_tokens_priority": 3e-05, + "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, + }, + "gpt-6-luna": { + "input_cost_per_token_above_272k_tokens_priority": 4e-07, + "output_cost_per_token_above_272k_tokens_priority": 1.5e-06, + "cache_read_input_token_cost_above_272k_tokens_priority": 4e-08, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07, + }, } EXPECTED = {**FLEX_LONG_CONTEXT, **PRIORITY_LONG_CONTEXT} @@ -90,4 +102,6 @@ TIERED_COST_CASES = [ ("gpt-5.6-terra", "priority", 8e-06, 3.6e-05), ("gpt-5.6-luna", "priority", 8e-07, 3.6e-06), ("gpt-6-astra", "priority", 4e-05, 0.00015), + ("gpt-6-sol", "priority", 8e-06, 3e-05), + ("gpt-6-luna", "priority", 4e-07, 1.5e-06), ] diff --git a/tests/test_litellm/test_pre_commit_lint.py b/tests/test_litellm/test_pre_commit_lint.py index e12da0833dc..2d7fa897536 100644 --- a/tests/test_litellm/test_pre_commit_lint.py +++ b/tests/test_litellm/test_pre_commit_lint.py @@ -9,6 +9,8 @@ from pathlib import Path import pytest +from tests._process_helpers import process_is_gone + ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts" / "pre_commit_lint.sh" WHOLE_TREE_RUFF = "run --no-sync ruff check --config ruff-tests.toml tests" @@ -343,14 +345,6 @@ def _wait_until(predicate: Callable[[], bool], timeout_seconds: float) -> bool: return predicate() -def _pid_gone(pid: int) -> bool: - try: - os.kill(pid, 0) - except ProcessLookupError: - return True - return False - - def test_interrupt_kills_background_jobs_and_removes_logs(tmp_path: Path) -> None: repo, bin_dir = _sandbox(tmp_path) hang_dir = tmp_path / "hang" @@ -372,7 +366,7 @@ def test_interrupt_kills_background_jobs_and_removes_logs(tmp_path: Path) -> Non os.killpg(proc.pid, signal.SIGINT) assert proc.wait(timeout=10) != 0 make_pid = int((hang_dir / "make.pid").read_text()) - assert _wait_until(lambda: _pid_gone(make_pid), 5) + assert process_is_gone(make_pid, within_seconds=5) assert _wait_until(lambda: not any(tmp_dir.iterdir()), 5), list(tmp_dir.iterdir()) finally: with suppress(ProcessLookupError, PermissionError): diff --git a/tests/test_litellm/test_process_helpers.py b/tests/test_litellm/test_process_helpers.py new file mode 100644 index 00000000000..00d4d07d71f --- /dev/null +++ b/tests/test_litellm/test_process_helpers.py @@ -0,0 +1,65 @@ +"""``process_is_gone`` has to say gone for every shape a killed process can take, and never for a live one.""" + +import os +import signal +import subprocess +import sys +from pathlib import Path +from typing import Final + +import pytest + +from tests._process_helpers import process_is_gone + +SLEEP_FOREVER: Final = (sys.executable, "-I", "-c", "import time; time.sleep(600)") + +LEAVE_A_ZOMBIE_BEHIND: Final = """ +import os, signal, subprocess, sys, time +grandchild = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(600)"]) +os.kill(grandchild.pid, signal.SIGKILL) +while open(f"/proc/{grandchild.pid}/stat").read().rpartition(")")[2].split()[0] != "Z": + time.sleep(0.01) +print(grandchild.pid, flush=True) +time.sleep(600) +""" + + +def test_a_live_process_is_not_gone() -> None: + child: Final = subprocess.Popen(SLEEP_FOREVER) + try: + assert not process_is_gone(child.pid, within_seconds=0.3) + finally: + child.kill() + child.wait() + + +def test_a_reaped_child_is_gone() -> None: + child: Final = subprocess.Popen(SLEEP_FOREVER) + child.kill() + child.wait() + assert process_is_gone(child.pid, within_seconds=1) + + +@pytest.mark.skipif(os.name == "nt", reason="zombies are a POSIX thing") +def test_an_unreaped_child_is_reaped_and_gone() -> None: + child: Final = subprocess.Popen(SLEEP_FOREVER) + os.kill(child.pid, signal.SIGKILL) + assert process_is_gone(child.pid, within_seconds=1) + with pytest.raises(ChildProcessError): + os.waitpid(child.pid, os.WNOHANG) + + +@pytest.mark.skipif(not Path("/proc").is_dir(), reason="needs procfs to see a zombie that is not our child") +def test_a_zombie_left_by_another_process_is_gone() -> None: + zombie_factory: Final = subprocess.Popen( + [sys.executable, "-I", "-c", LEAVE_A_ZOMBIE_BEHIND], stdout=subprocess.PIPE, text=True + ) + try: + assert zombie_factory.stdout is not None + zombie_pid: Final = int(zombie_factory.stdout.readline()) + with pytest.raises(ChildProcessError): + os.waitpid(zombie_pid, os.WNOHANG) + assert process_is_gone(zombie_pid, within_seconds=1) + finally: + zombie_factory.kill() + zombie_factory.wait() diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 0df6a181957..411377f29cf 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -9,7 +9,7 @@ import sys import threading import warnings from collections.abc import Awaitable, Callable, Mapping -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -850,36 +850,6 @@ async def test_arouter_async_get_healthy_deployments(): assert result[0]["litellm_params"]["model"] == "gpt-3.5-turbo" -@pytest.mark.asyncio -@patch("litellm.amoderation") -async def test_arouter_amoderation_with_credential_name(mock_amoderation): - """ - Test that router.amoderation passes litellm_credential_name to the underlying litellm.amoderation call - """ - mock_amoderation.return_value = AsyncMock() - - router = litellm.Router( - model_list=[ - { - "model_name": "text-moderation-stable", - "litellm_params": { - "model": "text-moderation-stable", - "litellm_credential_name": "my-custom-auth", - }, - }, - ], - ) - - await router.amoderation(input="I love everyone!", model="text-moderation-stable") - - mock_amoderation.assert_called_once() - call_kwargs = mock_amoderation.call_args[1] # Get the kwargs of the call - print( - "call kwargs for router.amoderation=", - json.dumps(call_kwargs, indent=4, default=str), - ) - assert call_kwargs["litellm_credential_name"] == "my-custom-auth" - assert call_kwargs["model"] == "text-moderation-stable" def test_arouter_test_team_model(): @@ -17620,3 +17590,242 @@ class TestMemberAutoRouterInference: monkeypatch.setitem(sys.modules, "fastapi", None) monkeypatch.delitem(sys.modules, "litellm.proxy.auth.auto_router_checks", raising=False) assert (await self._route(router, {"metadata": {"user_api_key_team_id": "router-team"}})).model == "restricted-model" + + +def _access_window_offsets(start_hours: float, end_hours: float, team_ids: list) -> dict: + now_utc = datetime.now(timezone.utc) + return { + "start": (now_utc + timedelta(hours=start_hours)).strftime("%H:%M"), + "end": (now_utc + timedelta(hours=end_hours)).strftime("%H:%M"), + "timezone": "UTC", + "team_ids": team_ids, + } + + +def _reserved_model_list(windows_for_reserved=None, windows_for_open=None) -> list: + reserved: dict = { + "model_name": "gpt-4o-ptu", + "litellm_params": {"model": "gpt-4o", "mock_response": "reserved"}, + "model_info": {"id": "reserved-deployment"}, + } + if windows_for_reserved is not None: + reserved["model_info"]["access_windows"] = windows_for_reserved + unreserved: dict = { + "model_name": "gpt-4o-ptu", + "litellm_params": {"model": "gpt-4o", "mock_response": "open"}, + "model_info": {"id": "open-deployment"}, + } + if windows_for_open is not None: + unreserved["model_info"]["access_windows"] = windows_for_open + return [reserved, unreserved] + + +def test_access_windows_hide_reserved_deployment_from_other_teams(): + router = Router( + model_list=_reserved_model_list( + windows_for_reserved=[_access_window_offsets(-1, 1, ["team-a"])], + ), + ) + _, deployments = router._common_checks_available_deployment( + model="gpt-4o-ptu", + request_kwargs={"metadata": {"user_api_key_team_id": "team-b"}}, + ) + assert [d["model_info"]["id"] for d in deployments] == ["open-deployment"] + + +def test_access_windows_raise_when_only_reserved_deployments_remain(): + router = Router(model_list=_reserved_model_list( + windows_for_reserved=[_access_window_offsets(-1, 1, ["team-a"])], + windows_for_open=[_access_window_offsets(-1, 1, ["team-a"])], + )[:1]) + for request_kwargs in ({"metadata": {"user_api_key_team_id": "team-b"}}, {}): + with pytest.raises(litellm.BadRequestError, match="reserved for another team"): + router._common_checks_available_deployment(model="gpt-4o-ptu", request_kwargs=request_kwargs) + _, deployments = router._common_checks_available_deployment( + model="gpt-4o-ptu", + request_kwargs={"metadata": {"user_api_key_team_id": "team-a"}}, + ) + assert [d["model_info"]["id"] for d in deployments] == ["reserved-deployment"] + + +def test_reserved_deployments_drop_strategy_markers_before_filtering(): + router = Router(model_list=_reserved_model_list()[:1]) + marker = {"model_name": "gpt-4o-ptu", "litellm_params": {"model": "auto_router/semantic"}} + reserved = { + "model_name": "gpt-4o-ptu", + "litellm_params": {"model": "gpt-4o"}, + "model_info": {"access_windows": [_access_window_offsets(-1, 1, ["team-a"])]}, + } + with pytest.raises(litellm.BadRequestError, match="reserved for another team"): + router._filter_reserved_deployments( + model="gpt-4o-ptu", + healthy_deployments=[marker, reserved], + request_team_id="team-b", + ) + + +def test_access_windows_invalid_timezone_fails_router_construction(): + with pytest.raises(ValueError, match=r"gpt-4o-ptu.*access_windows"): + Router( + model_list=[ + { + "model_name": "gpt-4o-ptu", + "litellm_params": {"model": "gpt-4o", "mock_response": "x"}, + "model_info": { + "access_windows": [ + { + "start": "22:00", + "end": "06:00", + "timezone": "Mars/Olympus", + "team_ids": ["team-a"], + } + ] + }, + } + ], + ) + + +def test_access_windows_inactive_window_leaves_deployments_available(): + router = Router( + model_list=_reserved_model_list( + windows_for_reserved=[_access_window_offsets(2, 3, ["team-a"])], + ), + ) + _, deployments = router._common_checks_available_deployment( + model="gpt-4o-ptu", + request_kwargs={"metadata": {"user_api_key_team_id": "team-b"}}, + ) + assert {d["model_info"]["id"] for d in deployments} == {"reserved-deployment", "open-deployment"} + + +def test_access_windows_apply_when_calling_by_model_id(): + router = Router(model_list=_reserved_model_list( + windows_for_reserved=[_access_window_offsets(-1, 1, ["team-a"])], + )) + with pytest.raises(litellm.BadRequestError, match="reserved for another team"): + router._common_checks_available_deployment( + model="reserved-deployment", + request_kwargs={"metadata": {"user_api_key_team_id": "team-b"}}, + ) + _, deployment = router._common_checks_available_deployment( + model="reserved-deployment", + request_kwargs={"metadata": {"user_api_key_team_id": "team-a"}}, + ) + assert deployment["model_info"]["id"] == "reserved-deployment" + + +def test_access_windows_apply_when_calling_by_litellm_model_name(): + router = Router( + model_list=[ + { + "model_name": "gpt-4o-ptu", + "litellm_params": { + "model": "openai/gpt-5.6-bypass-probe", + "mock_response": "reserved", + }, + "model_info": { + "id": "reserved-litellm-model", + "access_windows": [_access_window_offsets(-1, 1, ["team-a"])], + }, + } + ], + ) + with pytest.raises(litellm.BadRequestError, match="reserved for another team"): + router._common_checks_available_deployment( + model="openai/gpt-5.6-bypass-probe", + request_kwargs={"metadata": {"user_api_key_team_id": "team-b"}}, + ) + _, deployments = router._common_checks_available_deployment( + model="openai/gpt-5.6-bypass-probe", + request_kwargs={"metadata": {"user_api_key_team_id": "team-a"}}, + ) + assert [d["model_info"]["id"] for d in deployments] == ["reserved-litellm-model"] + + +def test_access_windows_apply_to_specific_deployment_calls(): + router = Router( + model_list=[ + { + "model_name": "gpt-4o-ptu", + "litellm_params": { + "model": "openai/gpt-5.6-specific-probe", + "mock_response": "reserved", + }, + "model_info": { + "id": "reserved-specific", + "access_windows": [_access_window_offsets(-1, 1, ["team-a"])], + }, + } + ], + ) + with pytest.raises(litellm.BadRequestError, match="reserved for another team"): + router._common_checks_available_deployment( + model="openai/gpt-5.6-specific-probe", + specific_deployment=True, + request_kwargs={"metadata": {"user_api_key_team_id": "team-b"}}, + ) + _, deployments = router._common_checks_available_deployment( + model="openai/gpt-5.6-specific-probe", + specific_deployment=True, + request_kwargs={"metadata": {"user_api_key_team_id": "team-a"}}, + ) + assert [d["model_info"]["id"] for d in deployments] == ["reserved-specific"] + + +def test_access_windows_apply_to_wildcard_early_resolve(): + router = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "mock_response": "reserved"}, + "model_info": { + "id": "reserved-wildcard", + "access_windows": [_access_window_offsets(-1, 1, ["team-a"])], + }, + } + ], + ) + with pytest.raises(litellm.BadRequestError, match="reserved for another team"): + router._common_checks_available_deployment( + model="openai/gpt-probe-wildcard", + request_kwargs={"metadata": {"user_api_key_team_id": "team-b"}}, + ) + _, deployments = router._common_checks_available_deployment( + model="openai/gpt-probe-wildcard", + request_kwargs={"metadata": {"user_api_key_team_id": "team-a"}}, + ) + assert [d["model_info"]["id"] for d in deployments] == ["reserved-wildcard"] + + +def test_access_windows_filter_reserved_deployments_method(): + router = Router(model_list=_reserved_model_list()) + reserved: dict = { + "model_info": { + "id": "reserved-deployment", + "access_windows": [_access_window_offsets(-1, 1, ["team-a"])], + } + } + open_deployment: dict = {"model_info": {"id": "open-deployment"}} + assert [ + d["model_info"]["id"] + for d in router._filter_reserved_deployments( + model="gpt-4o-ptu", + healthy_deployments=[reserved, open_deployment], + request_team_id="team-b", + ) + ] == ["open-deployment"] + with pytest.raises(litellm.BadRequestError, match="reserved for another team"): + router._filter_reserved_deployments( + model="gpt-4o-ptu", + healthy_deployments=[reserved], + request_team_id="team-b", + ) + assert [ + d["model_info"]["id"] + for d in router._filter_reserved_deployments( + model="gpt-4o-ptu", + healthy_deployments=[reserved, open_deployment], + request_team_id="team-a", + ) + ] == ["reserved-deployment", "open-deployment"] diff --git a/tests/test_litellm/test_router_exception_redaction.py b/tests/test_litellm/test_router_exception_redaction.py index 6754775db22..5b52496fc1c 100644 --- a/tests/test_litellm/test_router_exception_redaction.py +++ b/tests/test_litellm/test_router_exception_redaction.py @@ -26,8 +26,8 @@ Five leak sites are gated in `litellm/router.py`: 1. Deployment timeout debug after `litellm.Timeout` 2. ContextWindowExceededError fallback hint 3. ContentPolicyViolationError fallback hint -4. "No fallback model group found for..." when fallbacks dict misses -5. "Received Model Group=...\\nAvailable Model Group Fallbacks=..." +4. "no fallback model group was found" when fallbacks dict misses +5. "model group '...' failed with the error above" plus the fallback outcome (always fires on terminal raise from the fallback orchestrator) Site 5 is the broadest — it fires for every failing call that goes @@ -42,8 +42,9 @@ import pytest import litellm from litellm import Router -_RECEIVED_MODEL_GROUP_PHRASE = "Received Model Group=" -_AVAILABLE_FALLBACKS_PHRASE = "Available Model Group Fallbacks=" +_RECEIVED_MODEL_GROUP_PHRASE = "failed with the error above" +_AVAILABLE_FALLBACKS_PHRASE = "No fallback was attempted" +_NO_FALLBACK_GROUP_PHRASE = "no fallback model group was found" _CONTEXT_WINDOW_HINT_PHRASE = "context_window_fallbacks=" _INTERNAL_MODEL_GROUP_NAME = "all-anthropic/claude-secret-internal" _FALLBACK_CREDENTIAL = "sk-INLINEFALLBACKSECRET1234567890" @@ -124,7 +125,7 @@ def test_flag_defaults_on(): assert litellm.expose_router_debug_in_errors is True -# --- Site 5: "Received Model Group=..." on terminal raise -------------------- +# --- Site 5: fallback outcome on terminal raise -------------------- @pytest.mark.asyncio @@ -192,7 +193,7 @@ async def test_flag_on_shows_context_window_fallback_hint(monkeypatch: pytest.Mo assert _INTERNAL_MODEL_GROUP_NAME in msg, msg -# --- Site 4: "No fallback model group found..." when fallbacks miss --------- +# --- Site 4: "no fallback model group was found" when fallbacks miss --------- @pytest.mark.asyncio @@ -221,7 +222,7 @@ async def test_flag_off_does_not_leak_when_no_fallback_group_found(monkeypatch: messages=[{"role": "user", "content": "hi"}], ) msg = excinfo.value.message - assert "No fallback model group found" not in msg, msg + assert _NO_FALLBACK_GROUP_PHRASE not in msg, msg assert "some-other-group" not in msg, msg assert _INTERNAL_MODEL_GROUP_NAME not in msg, msg @@ -250,8 +251,12 @@ async def test_flag_on_shows_when_no_fallback_group_found(monkeypatch: pytest.Mo messages=[{"role": "user", "content": "hi"}], ) msg = excinfo.value.message - assert "No fallback model group found" in msg, msg - assert _INTERNAL_MODEL_GROUP_NAME in msg, msg + assert _NO_FALLBACK_GROUP_PHRASE in msg, msg + assert f"model group '{_INTERNAL_MODEL_GROUP_NAME}' failed with the error above" in msg, msg + assert "Fallbacks are configured for: some-other-group" in msg, msg + assert "not retried on another model" in msg, msg + assert _AVAILABLE_FALLBACKS_PHRASE not in msg, msg + assert msg.count("failed with the error above") == 1, msg # --- Site 1: Deployment timeout debug on litellm.Timeout -------------------- @@ -349,6 +354,30 @@ async def test_flag_on_shows_content_policy_fallback_hint(monkeypatch: pytest.Mo assert _INTERNAL_MODEL_GROUP_NAME in msg, msg +@pytest.mark.asyncio +async def test_flag_on_explains_failed_content_policy_fallback(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "expose_router_debug_in_errors", True) + router = Router( + model_list=[ + {"model_name": _INTERNAL_MODEL_GROUP_NAME, "litellm_params": {"model": "gpt-4o", "api_key": "key"}}, + {"model_name": "policy-safe-group", "litellm_params": {"model": "gpt-4o", "api_key": "key"}}, + ], + content_policy_fallbacks=[{_INTERNAL_MODEL_GROUP_NAME: ["policy-safe-group"]}], + num_retries=0, + ) + with pytest.raises(litellm.ContentPolicyViolationError) as excinfo: + await router.acompletion( + model=_INTERNAL_MODEL_GROUP_NAME, + messages=[{"role": "user", "content": "hi"}], + mock_response=_content_policy_error(), + ) + msg = excinfo.value.message + assert f"model group '{_INTERNAL_MODEL_GROUP_NAME}' failed with the error above" in msg, msg + assert "Fallback to policy-safe-group also failed: " in msg, msg + assert _AVAILABLE_FALLBACKS_PHRASE not in msg, msg + assert msg.count("failed with the error above") == 1, msg + + # --- Credential masking: raw provider keys never leak, either flag state ---- @@ -387,7 +416,7 @@ async def test_flag_on_masks_fallback_credentials(monkeypatch: pytest.MonkeyPatc async def test_flag_on_scrubs_credential_from_inner_fallback_exception_string(monkeypatch: pytest.MonkeyPatch): """If the fallback attempt itself raises an exception whose message embeds a raw provider credential (e.g. a provider SDK echoing back the api_key it was - called with), that string is re-embedded via `Error doing the fallback: ...` + called with), that string is re-embedded via `Fallback to ... also failed: ...` on the terminal raise. The router must scrub known secret patterns from it. The primary fails with a benign rate-limit; the fallback deployment fails with an exception whose text contains the secret.""" @@ -423,6 +452,9 @@ async def test_flag_on_scrubs_credential_from_inner_fallback_exception_string(mo messages=[{"role": "user", "content": "hi"}], ) msg = excinfo.value.message - assert "Error doing the fallback:" in msg, msg + assert f"model group '{_INTERNAL_MODEL_GROUP_NAME}' failed with the error above" in msg, msg + assert "Fallback to fallback-group also failed: " in msg, msg + assert "content_filter_policy" in msg, msg + assert msg.count("failed with the error above") == 1, msg assert inner_secret not in msg, msg assert "REDACTED" in msg, msg diff --git a/tests/test_litellm/test_together_ai_model_metadata.py b/tests/test_litellm/test_together_ai_model_metadata.py index 7176ba4f219..dccba3d66f9 100644 --- a/tests/test_litellm/test_together_ai_model_metadata.py +++ b/tests/test_litellm/test_together_ai_model_metadata.py @@ -95,15 +95,6 @@ def _successor(info: dict[str, object]) -> str | None: return successor if isinstance(successor, str) else None -def test_together_successor_metadata_points_at_known_models(cost_map: CostMap): - successors = { - model: successor - for model, info in cost_map.items() - if model.startswith("together_ai/") and (successor := _successor(info)) is not None - } - assert len(successors) >= 10 - for model, successor in successors.items(): - assert successor in cost_map, f"{model} names successor {successor} that is not in the map" def test_together_backup_cost_map_in_sync(cost_map: CostMap): diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 56edcfbd22b..5ac8a71d208 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1,6 +1,8 @@ import asyncio +import base64 import contextlib import contextvars +import io import json import logging import os @@ -9,7 +11,8 @@ import threading from collections.abc import Callable, Iterator from concurrent.futures import Future, ThreadPoolExecutor from datetime import datetime, timedelta, timezone -from typing import Final +from pathlib import PurePath +from typing import Final, cast from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -19,9 +22,6 @@ from jsonschema import validate import litellm from litellm._internal_context import is_internal_call -from litellm.caching.caching import Cache -from litellm.caching.caching_handler import _PENDING_CACHE_WRITES -from litellm.constants import DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT from litellm._logging import ( CorrelationContextFilter, JsonFormatter, @@ -29,14 +29,18 @@ from litellm._logging import ( trace_id_var, verbose_logger, ) +from litellm.caching.caching import Cache +from litellm.caching.caching_handler import _PENDING_CACHE_WRITES +from litellm.constants import DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.thread_pool_executor import executor as logging_executor from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.proxy.utils import is_valid_api_key -from litellm.types.router import CredentialLiteLLMParams, GenericLiteLLMParams from litellm.types.integrations.custom_logger import HEADROOM_CONVERTED_STREAM_KEY +from litellm.types.router import CredentialLiteLLMParams, GenericLiteLLMParams from litellm.types.utils import ( + ADDRESSED_RESPONSE_ID_FIELD, CallTypes, Choices, Delta, @@ -47,7 +51,6 @@ from litellm.types.utils import ( PromptTokensDetailsWrapper, StreamingChoices, Usage, - ADDRESSED_RESPONSE_ID_FIELD, all_litellm_params, bedrock_batch_litellm_params, ) @@ -179,6 +182,14 @@ def test_potential_model_names_keeps_provider_prefixed_candidate(): assert bare["provider_prefixed_model_name"] == bare["combined_model_name"] == "perplexity/glm-5.2" +@pytest.mark.parametrize("capability", [True, False, None]) +def test_get_model_info_anthropic_compaction( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, capability: bool | None +) -> None: + monkeypatch.setitem(litellm.model_cost["claude-sonnet-5"], "supports_anthropic_compaction", capability) + assert litellm.get_model_info("claude-sonnet-5")["supports_anthropic_compaction"] is capability + + def test_get_model_info_strips_openai_finetune_ids_without_a_custom_suffix(local_model_cost_map): info = litellm.get_model_info(model="ft:gpt-4o-2024-08-06:my-org::abc123", custom_llm_provider="openai") assert info["key"] == "ft:gpt-4o-2024-08-06" @@ -733,6 +744,8 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_creation_input_token_cost_above_272k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_272k_tokens_flex": {"type": "number"}, "cache_creation_input_token_cost_above_272k_tokens_priority": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens_batches": {"type": "number"}, + "cache_creation_input_token_cost_batches": {"type": "number"}, "cache_creation_input_token_cost_flex": {"type": "number"}, "cache_creation_input_token_cost_priority": {"type": "number"}, "cache_read_input_token_cost": {"type": "number"}, @@ -742,6 +755,8 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_read_input_token_cost_above_272k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_272k_tokens_flex": {"type": "number"}, "cache_read_input_token_cost_above_512k_tokens": {"type": "number"}, + "cache_read_input_token_cost_batches": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens_batches": {"type": "number"}, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"}, "cache_read_input_audio_token_cost": {"type": "number"}, "cache_read_input_image_token_cost": {"type": "number"}, @@ -769,12 +784,14 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_token_priority": {"type": "number"}, "input_cost_per_token_above_200k_tokens_priority": {"type": "number"}, "input_cost_per_token_above_272k_tokens_priority": {"type": "number"}, + "input_cost_per_token_above_272k_tokens_batches": {"type": "number"}, "input_cost_per_token_above_272k_tokens_flex": {"type": "number"}, "input_cost_per_audio_token_priority": {"type": "number"}, "output_cost_per_token_flex": {"type": "number"}, "output_cost_per_token_priority": {"type": "number"}, "output_cost_per_token_above_200k_tokens_priority": {"type": "number"}, "output_cost_per_token_above_272k_tokens_priority": {"type": "number"}, + "output_cost_per_token_above_272k_tokens_batches": {"type": "number"}, "output_cost_per_token_above_272k_tokens_flex": {"type": "number"}, "regional_endpoint_uplift_multiplier": {"type": "number"}, "regional_processing_uplift_multiplier_eu": {"type": "number"}, @@ -862,6 +879,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "source": {"type": "string"}, "comment": {"type": "string"}, "supports_assistant_prefill": {"type": "boolean"}, + "supports_anthropic_compaction": {"type": "boolean"}, "supports_audio_input": {"type": "boolean"}, "supports_audio_output": {"type": "boolean"}, "gemini_native_audio": {"type": "boolean"}, @@ -1448,12 +1466,6 @@ class TestProxyFunctionCalling: ("gemini/gemini-2.5-pro", "litellm_proxy/gemini/gemini-2.5-pro", True), ("gemini/gemini-2.5-flash", "litellm_proxy/gemini/gemini-2.5-flash", True), # Groq models (mixed support) - ("groq/gemma-7b-it", "litellm_proxy/groq/gemma-7b-it", True), - ( - "groq/llama-3.3-70b-versatile", - "litellm_proxy/groq/llama-3.3-70b-versatile", - True, - ), # Cohere models (generally don't support function calling) ("command-nightly", "litellm_proxy/command-nightly", False), ], @@ -1624,7 +1636,6 @@ class TestProxyFunctionCalling: # For now, we expect False (current behavior), but document the limitation assert proxy_result is False, f"Current limitation: {proxy_model_with_hints} returns False without inference" - def test_litellm_utils_supports_function_calling_import(self): """Test that supports_function_calling can be imported from litellm.utils.""" try: @@ -1644,7 +1655,6 @@ class TestProxyFunctionCalling: except Exception as e: pytest.fail(f"Failed to access litellm.supports_function_calling: {e}") - def test_edge_cases_and_malformed_proxy_models(self): """Test edge cases and malformed proxy model names.""" test_cases = [ @@ -3027,6 +3037,45 @@ class TestExtraBodyCannotOverrideModel: assert result["extra_body"] == {"top_k": 5}, result + def test_nested_drop_paths_do_not_break_extra_body_filtering(self) -> None: + from litellm.utils import add_provider_specific_params_to_optional_params + + result = add_provider_specific_params_to_optional_params( + optional_params={}, + passed_params={ + "model": "hosted_vllm/my-vllm-model", + "extra_body": {"model": "hosted_vllm/other", "top_k": 5, "kept": True}, + }, + custom_llm_provider="hosted_vllm", + openai_params=["model", "temperature"], + additional_drop_params=[["tools", "function", "strict"], "top_k"], + ) + + assert result == {"extra_body": {"kept": True}}, result + + def test_a_list_entry_does_not_break_a_supported_nested_drop_path(self) -> None: + def tools() -> list[dict]: + return [ + { + "type": "function", + "function": {"name": "f", "custom_marker": "LEAK", "parameters": {"type": "object"}}, + } + ] + + untouched = litellm.get_optional_params( + model="my-vllm-model", custom_llm_provider="hosted_vllm", tools=tools() + ) + assert untouched["tools"][0]["function"]["custom_marker"] == "LEAK", untouched + + result = litellm.get_optional_params( + model="my-vllm-model", + custom_llm_provider="hosted_vllm", + tools=tools(), + additional_drop_params=["tools[*].function.custom_marker", ["tools", "function", "custom_marker"]], + ) + + assert "custom_marker" not in result["tools"][0]["function"], result + class TestDropParamsWithPromptCacheKey: """ @@ -5146,6 +5195,102 @@ async def test_wrapper_async_fires_post_call_failure_deployment_hook_on_internal assert isinstance(recorder.calls[0][1], litellm.AuthenticationError) +def _budget_reservation(callback_bound: bool = False) -> dict: + return {"reserved_cost": 0.5, "entries": [], "finalized": False, "callback_bound": callback_bound} + + +_BUDGET_RESERVATION_CALL_KWARGS: Final = {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]} +_BUDGET_RESERVATION_REFUSAL: Final = litellm.AuthenticationError( + message="bad key", llm_provider="openai", model="gpt-4o" +) + + +@pytest.mark.asyncio +async def test_wrapper_async_claims_the_budget_reservation_for_the_cost_callback() -> None: + reservation = _budget_reservation() + + await litellm.acompletion( + **_BUDGET_RESERVATION_CALL_KWARGS, + mock_response="ok", + metadata={"user_api_key_budget_reservation": reservation}, + ) + + assert reservation["callback_bound"] is True + + +@pytest.mark.asyncio +async def test_wrapper_async_claims_the_budget_reservation_before_the_stream_is_consumed() -> None: + reservation = _budget_reservation() + + stream = await litellm.acompletion( + **_BUDGET_RESERVATION_CALL_KWARGS, + mock_response="ok", + stream=True, + metadata={"user_api_key_budget_reservation": reservation}, + ) + + assert reservation["callback_bound"] is True + async for _ in stream: + pass + + +@pytest.mark.asyncio +async def test_wrapper_async_claims_the_budget_reservation_a_supplied_logging_object_already_saw() -> None: + reservation = _budget_reservation() + logging_obj, kwargs = litellm.utils.function_setup( + original_function="acompletion", + rules_obj=litellm.utils.Rules(), + start_time=datetime.now(), + **_BUDGET_RESERVATION_CALL_KWARGS, + litellm_call_id="proxy-pre-call-setup", + metadata={"user_api_key_budget_reservation": reservation}, + ) + assert reservation["callback_bound"] is False + + await litellm.acompletion(**kwargs, litellm_logging_obj=logging_obj, mock_response="ok") + + assert reservation["callback_bound"] is True + + +@pytest.mark.asyncio +async def test_wrapper_async_hands_the_budget_reservation_back_when_the_call_fails() -> None: + reservation = _budget_reservation() + + with pytest.raises(litellm.AuthenticationError): + await litellm.acompletion( + **_BUDGET_RESERVATION_CALL_KWARGS, + mock_response=_BUDGET_RESERVATION_REFUSAL, + metadata={"user_api_key_budget_reservation": reservation}, + ) + + assert reservation["callback_bound"] is False + + +@pytest.mark.asyncio +async def test_wrapper_async_leaves_the_budget_reservation_alone_on_internal_calls() -> None: + claimed_by_the_outer_call = _budget_reservation(callback_bound=True) + never_claimed = _budget_reservation() + + token = is_internal_call.set(True) + try: + await litellm.acompletion( + **_BUDGET_RESERVATION_CALL_KWARGS, + mock_response="ok", + metadata={"user_api_key_budget_reservation": never_claimed}, + ) + with pytest.raises(litellm.AuthenticationError): + await litellm.acompletion( + **_BUDGET_RESERVATION_CALL_KWARGS, + mock_response=_BUDGET_RESERVATION_REFUSAL, + metadata={"user_api_key_budget_reservation": claimed_by_the_outer_call}, + ) + finally: + is_internal_call.reset(token) + + assert never_claimed["callback_bound"] is False + assert claimed_by_the_outer_call["callback_bound"] is True + + @pytest.mark.asyncio async def test_wrapper_async_does_not_fire_failure_hook_for_pre_call_budget_error( monkeypatch: pytest.MonkeyPatch, @@ -6057,3 +6202,111 @@ def test_calculate_max_parallel_requests_precedence( ) == expected ) + + +class _NamedStream(io.BytesIO): + def __init__(self, name: str | int) -> None: + super().__init__(b"%PDF-1.4 secret document body") + self.name = name + + +def _logged_request_messages(original_function: str, *args: object, **kwargs: object) -> object: + logging_obj, _ = litellm.utils.function_setup( + original_function, + litellm.utils.Rules(), + datetime.now(), + *args, + litellm_call_id="request-text-call", + **kwargs, + ) + return logging_obj.messages + + +@pytest.mark.parametrize( + ("original_function", "args", "kwargs", "expected"), + [ + ("search", (), {"query": "Eiffel Tower"}, "Eiffel Tower"), + ("asearch", ("Eiffel Tower",), {}, "Eiffel Tower"), + ("asearch", (), {"query": ["Eiffel Tower", "Louvre"]}, "Eiffel Tower\nLouvre"), + ("asearch", (), {"query": ["Eiffel Tower", 7, None]}, "Eiffel Tower"), + ("image_edit", (), {"prompt": "make it blue", "image": b"png"}, "make it blue"), + ("aimage_edit", (b"png", "make it blue"), {}, "make it blue"), + ( + "aocr", + (), + {"document": {"type": "document_url", "document_url": "https://x.test/a.pdf"}}, + "https://x.test/a.pdf", + ), + ( + "ocr", + ("mistral-ocr-latest", {"type": "image_url", "image_url": "https://x.test/a.png"}), + {}, + "https://x.test/a.png", + ), + ( + "aocr", + (), + {"document": {"type": "document_url", "document_url": "data:application/pdf;base64,JVBERi0xLjQ="}}, + "data:application/pdf;base64 (12 chars)", + ), + ( + "aocr", + (), + {"document": {"type": "image_url", "image_url": "https://x.test/a,b.png"}}, + "https://x.test/a,b.png", + ), + ( + "aocr", + (), + {"document": {"type": "file", "file": PurePath("/tmp/hello.pdf"), "mime_type": "application/pdf"}}, + "file (application/pdf) hello.pdf", + ), + ("aocr", (), {"document": {"type": "document_url", "document_url": ""}}, ""), + ("aocr", (), {"document": {"type": "file", "file": b"%PDF"}}, "file 4 bytes"), + ("aocr", (), {"document": {"type": "file", "file": io.BytesIO(b"%PDF")}}, "file"), + ("aocr", (), {"document": {"type": "file", "file": _NamedStream("/tmp/scan.pdf")}}, "file scan.pdf"), + ( + "aocr", + (), + {"document": {"type": "file", "file": _NamedStream(3), "mime_type": "application/pdf"}}, + "file (application/pdf)", + ), + ("aocr", (), {"document": "not-a-document"}, "default-message-value"), + ], +) +def test_function_setup_logs_the_search_query_edit_prompt_and_ocr_document_summary_as_the_request( + original_function: str, args: tuple[object, ...], kwargs: dict[str, object], expected: str +) -> None: + assert _logged_request_messages(original_function, *args, **kwargs) == [{"role": "user", "content": expected}] + + +def test_search_with_a_mixed_type_query_list_still_reaches_its_own_validation_error() -> None: + mixed_query: Final = cast(list[str], ["Eiffel Tower", 7]) # cast-ok: the invalid list is the point of the test + + with pytest.raises(litellm.APIConnectionError, match="All items in query list must be strings"): + litellm.search(query=mixed_query, search_provider="duckduckgo") + + +def test_function_setup_never_logs_the_ocr_file_bytes() -> None: + content: Final = b"%PDF-1.4 secret document body" + logged: Final = _logged_request_messages("aocr", document={"type": "file", "file": content}) + + assert logged == [{"role": "user", "content": "file 29 bytes"}] + + +def test_function_setup_leaves_the_ocr_file_stream_unread_and_never_logs_its_bytes() -> None: + stream: Final = _NamedStream("/tmp/scan.pdf") + logged: Final = _logged_request_messages("aocr", document={"type": "file", "file": stream}) + + assert logged == [{"role": "user", "content": "file scan.pdf"}] + assert stream.tell() == 0 + + +def test_function_setup_never_logs_the_ocr_data_uri_payload() -> None: + payload: Final = base64.b64encode(b"%PDF-1.4 secret document body").decode() + logged: Final = _logged_request_messages( + "aocr", document={"type": "document_url", "document_url": f"data:application/pdf;base64,{payload}"} + ) + + assert logged == [{"role": "user", "content": f"data:application/pdf;base64 ({len(payload)} chars)"}] + assert payload not in str(logged) diff --git a/tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py b/tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py index 72e98711f0c..2d5936fabdb 100644 --- a/tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py +++ b/tests/test_litellm/test_vertex_ai_xai_grok_prompt_caching_metadata.py @@ -13,16 +13,18 @@ GROK_KEY_PREFIXES: Final = ("vertex_ai/xai/grok-", "azure_ai/grok-", "xai/grok-" @pytest.mark.usefixtures("local_model_cost_map") def test_grok_models_with_cache_read_price_advertise_prompt_caching() -> None: - cached_grok_models = tuple( + cached_grok_models: Final = tuple( key for key, entry in litellm.model_cost.items() if key.startswith(GROK_KEY_PREFIXES) and entry.get("cache_read_input_token_cost") ) assert cached_grok_models, "expected at least one grok model with a cache read price" - missing_flag = tuple(key for key in cached_grok_models if supports_prompt_caching(model=key) is not True) + missing_flag: Final = tuple( + key for key in cached_grok_models if get_model_info(model=key).get("supports_prompt_caching") is not True + ) assert missing_flag == (), ( - f"grok models with cache_read_input_token_cost fail supports_prompt_caching: {missing_flag}" + f"grok models with cache_read_input_token_cost fail get_model_info supports_prompt_caching: {missing_flag}" ) @@ -31,8 +33,14 @@ def test_vertex_ai_grok_4_6_supports_prompt_caching_via_get_model_info() -> None routed_model, provider, _, _ = get_llm_provider(model=MODEL) assert (routed_model, provider) == ("xai/grok-4.6", "vertex_ai") - info = get_model_info(model=routed_model, custom_llm_provider=provider) - assert info["litellm_provider"] == "vertex_ai" - assert info.get("supports_prompt_caching") is True + routed_info: Final = get_model_info(model=routed_model, custom_llm_provider=provider) + assert routed_info["litellm_provider"] == "vertex_ai" + assert routed_info.get("supports_prompt_caching") is True + assert routed_info.get("cache_read_input_token_cost") + + catalog_info: Final = get_model_info(model=MODEL) + assert catalog_info["key"] == MODEL + assert catalog_info.get("supports_prompt_caching") is True + assert catalog_info.get("cache_read_input_token_cost") assert supports_prompt_caching(model=MODEL) is True diff --git a/tests/test_litellm/test_xai_responses_auto_routing.py b/tests/test_litellm/test_xai_responses_auto_routing.py index d405ea1e6c6..c31f599dd79 100644 --- a/tests/test_litellm/test_xai_responses_auto_routing.py +++ b/tests/test_litellm/test_xai_responses_auto_routing.py @@ -46,34 +46,6 @@ class TestXAIResponsesAutoRouting: assert model_info.get("mode") != "responses" assert updated_model == model - def test_responses_api_bridge_check_with_tools(self): - """Test that with tools, xAI automatically routes to Responses API""" - model = "grok-3" - custom_llm_provider = "xai" - tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the weather", - "parameters": { - "type": "object", - "properties": {"location": {"type": "string"}}, - }, - }, - } - ] - web_search_options = None - - model_info, updated_model = responses_api_bridge_check( - model=model, - custom_llm_provider=custom_llm_provider, - web_search_options=web_search_options, - ) - - # Should auto-route to responses mode when tools are present - assert model_info.get("mode") == "chat" - assert updated_model == model def test_responses_api_bridge_check_with_empty_tools(self): """Test that with empty tools list, xAI does not route to Responses API""" @@ -134,57 +106,8 @@ class TestXAIResponsesAutoRouting: assert model_info.get("mode") == "responses" assert updated_model == "grok-3" # prefix removed - def test_responses_api_bridge_check_with_code_interpreter_tool(self): - """Test auto-routing with code_interpreter tool""" - model = "grok-3" - custom_llm_provider = "xai" - tools = [{"type": "code_interpreter"}] - web_search_options = None - model_info, updated_model = responses_api_bridge_check( - model=model, - custom_llm_provider=custom_llm_provider, - web_search_options=web_search_options, - ) - # Should auto-route with code_interpreter tool - assert model_info.get("mode") == "chat" - assert updated_model == model - def test_responses_api_bridge_check_with_web_search_tool(self): - """Test auto-routing with web_search tool""" - model = "grok-4" - custom_llm_provider = "xai" - tools = [ - {"type": "web_search", "filters": {"allowed_domains": ["wikipedia.org"]}} - ] - web_search_options = None - - model_info, updated_model = responses_api_bridge_check( - model=model, - custom_llm_provider=custom_llm_provider, - web_search_options=web_search_options, - ) - - # Should auto-route with web_search tool - assert model_info.get("mode") == "chat" - assert updated_model == model - - def test_responses_api_bridge_check_with_x_search_tool(self): - """Test auto-routing with x_search tool""" - model = "grok-4" - custom_llm_provider = "xai" - tools = [{"type": "x_search", "allowed_x_handles": ["@elonmusk"]}] - web_search_options = None - - model_info, updated_model = responses_api_bridge_check( - model=model, - custom_llm_provider=custom_llm_provider, - web_search_options=web_search_options, - ) - - # Should auto-route with x_search tool - assert model_info.get("mode") == "chat" - assert updated_model == model def test_responses_api_bridge_check_with_web_search_options(self): """Test auto-routing with web_search_options""" diff --git a/tests/test_litellm/types/test_router.py b/tests/test_litellm/types/test_router.py index 7fcb76b638f..4d4c326d1ca 100644 --- a/tests/test_litellm/types/test_router.py +++ b/tests/test_litellm/types/test_router.py @@ -91,7 +91,7 @@ def test_pricing_strings_are_coerced_to_float(): def test_invalid_pricing_is_rejected(): - with pytest.raises(ValueError, match='validation error for ModelInfo'): + with pytest.raises(ValueError, match="validation error for ModelInfo"): ModelInfo(id="x", input_cost_per_token="free") @@ -118,7 +118,9 @@ def test_drop_params_ignores_non_flag_non_string_values_with_a_warning(value, ca assert f"drop_params={value!r} is not a flag value" in caplog.text -@pytest.mark.parametrize("value", [True, "true", None, "os.environ/DROP_PARAMS", "v2:gcm:ciphertext-from-a-pre-fix-row"]) +@pytest.mark.parametrize( + "value", [True, "true", None, "os.environ/DROP_PARAMS", "v2:gcm:ciphertext-from-a-pre-fix-row"] +) def test_drop_params_flags_and_strings_log_nothing(value, caplog): with caplog.at_level(logging.WARNING, logger="LiteLLM"): GenericLiteLLMParams(drop_params=value) @@ -146,3 +148,88 @@ def test_aws_session_tags_round_trip_as_sts_shaped_pairs(): def test_aws_session_tags_reject_shapes_sts_would_refuse(aws_session_tags): with pytest.raises(ValidationError, match="aws_session_tags"): LiteLLM_Params(model="bedrock/anthropic.claude-opus-5", aws_session_tags=aws_session_tags) + + +def test_provider_affinity_header_is_normalized(): + params = LiteLLM_Params( + model="openai/gpt-4o-mini", + provider_affinity_header="X-Conversation-Id", + ) + + assert params.provider_affinity_header == "X-Conversation-Id" + assert params.model_dump(exclude_none=True)["provider_affinity_header"] == "X-Conversation-Id" + + +@pytest.mark.parametrize( + "header", + [ + "Authorization", + "Proxy-Authorization", + "Cookie", + "Set-Cookie", + "Host", + "Content-Length", + "Content-Type", + "X-API-Key", + ], +) +def test_provider_affinity_header_rejects_sensitive_or_transport_headers(header: str): + with pytest.raises(ValueError, match="provider_affinity_header"): + LiteLLM_Params( + model="openai/gpt-4o-mini", + provider_affinity_header=header, + ) + + +@pytest.mark.parametrize("header", ["", "X Conversation Id", "X-Conversation-Id\r\nInjected: true"]) +def test_provider_affinity_header_rejects_invalid_header_names(header: str): + with pytest.raises(ValueError, match="provider_affinity_header"): + LiteLLM_Params( + model="openai/gpt-4o-mini", + provider_affinity_header=header, + ) + + +def test_model_info_parses_access_windows_time_strings(): + import datetime + + info = ModelInfo( + id="x", + access_windows=[ + { + "start": "22:00", + "end": "06:00", + "timezone": "America/New_York", + "team_ids": ["team-nightly"], + } + ], + ) + window = info.access_windows[0] + assert window.start == datetime.time(22, 0) + assert window.end == datetime.time(6, 0) + assert window.timezone == "America/New_York" + assert window.team_ids == ("team-nightly",) + + +@pytest.mark.parametrize( + "access_windows", + [ + [{"start": "25:00", "end": "06:00", "timezone": "UTC", "team_ids": ["t"]}], + [{"start": "22:00", "end": "06:00", "timezone": "Mars/Olympus", "team_ids": ["t"]}], + [{"start": "22:00", "end": "06:00", "timezone": "UTC", "team_ids": []}], + ], + ids=["invalid-time", "unknown-timezone", "empty-team-ids"], +) +def test_model_info_rejects_invalid_access_windows(access_windows): + with pytest.raises(ValidationError): + ModelInfo(id="x", access_windows=access_windows) + + +def test_model_info_rejects_offset_aware_access_window_times(): + with pytest.raises(ValidationError): + ModelInfo( + id="x", + access_windows=[ + {"start": "22:00+05:00", "end": "06:00", "timezone": "UTC", "team_ids": ["t"]} + ], + ) diff --git a/tests/test_litellm/types/test_types_utils.py b/tests/test_litellm/types/test_types_utils.py index 5f44ba1773e..0a8c9414a0d 100644 --- a/tests/test_litellm/types/test_types_utils.py +++ b/tests/test_litellm/types/test_types_utils.py @@ -1,9 +1,16 @@ +import json from typing import Final import pytest -from litellm.types.utils import HiddenParams, all_litellm_params, text_tokens_without_nested_reasoning +from litellm.types.utils import ( + HiddenParams, + ImageObject, + ImageResponse, + all_litellm_params, + text_tokens_without_nested_reasoning, +) def test_rust_is_a_known_litellm_param(): @@ -763,13 +770,70 @@ def test_delta_function_tool_call_unchanged_by_custom_support(): def test_image_response_keeps_background(): """https://github.com/BerriAI/litellm/issues/38649""" - from litellm.types.utils import ImageResponse - response = ImageResponse(created=1, data=[{"b64_json": "aGk="}], background="transparent", output_format="png") assert response.background == "transparent" assert response.model_dump()["background"] == "transparent" +def test_image_response_serialization_honors_dump_options(): + response: Final = ImageResponse( + data=[ + ImageObject( + url="https://example.com/image.png", + provider_specific_fields={"width": 1024, "height": 1536, "content_type": "image/png"}, + ) + ] + ) + expected: Final = [ + { + "url": "https://example.com/image.png", + "provider_specific_fields": {"width": 1024, "height": 1536, "content_type": "image/png"}, + } + ] + assert response.model_dump(exclude_none=True)["data"] == expected + assert json.loads(response.model_dump_json(exclude_none=True))["data"] == expected + assert response.model_dump()["data"][0]["provider_specific_fields"] == expected[0]["provider_specific_fields"] + assert "url" not in response.model_dump(exclude={"data": {0: {"url"}}})["data"][0] + assert response.model_dump(include={"data": {"__all__": {"url"}}})["data"] == [ + {"url": "https://example.com/image.png"} + ] + assert response.model_dump(include={"data": {0: True}})["data"] == [ + { + "b64_json": None, + "revised_prompt": None, + "url": "https://example.com/image.png", + "provider_specific_fields": {"width": 1024, "height": 1536, "content_type": "image/png"}, + } + ] + assert response.model_dump(exclude={"data": {0: True}})["data"] == [] + + two_image_response: Final = ImageResponse( + data=[ + ImageObject(url="https://example.com/image.png"), + ImageObject(url="https://example.com/second-image.png"), + ] + ) + assert two_image_response.model_dump(exclude={"data": {1}})["data"] == [ + { + "b64_json": None, + "revised_prompt": None, + "url": "https://example.com/image.png", + "provider_specific_fields": None, + } + ] + assert two_image_response.model_dump(exclude={"data": {-1}})["data"] == [ + { + "b64_json": None, + "revised_prompt": None, + "url": "https://example.com/image.png", + "provider_specific_fields": None, + } + ] + assert two_image_response.model_dump(include={"data": {-1: {"url"}}})["data"] == [ + {"url": "https://example.com/second-image.png"} + ] + + @pytest.mark.parametrize( ("completion_tokens", "text_tokens", "reasoning_tokens", "other_modality_tokens", "expected_text_tokens"), ( diff --git a/tests/test_litellm_rust/messages/test_callbacks.py b/tests/test_litellm_rust/messages/test_callbacks.py index b55bc47d640..d5dc5437398 100644 --- a/tests/test_litellm_rust/messages/test_callbacks.py +++ b/tests/test_litellm_rust/messages/test_callbacks.py @@ -5,7 +5,11 @@ import pytest import litellm from litellm.integrations.custom_logger import CustomLogger +from litellm.rust_bridge import catalog +from litellm.rust_bridge.catalog import Route, RouteRule +from litellm.rust_bridge.configuration import Rollout from tests.test_litellm_rust.support.callback_recorder import RecordingLogger, drain_logging +from tests.test_litellm_rust.support.isolation import rebound from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec from tests.test_litellm_rust.support.requests import ( MESSAGES, @@ -20,6 +24,12 @@ pytestmark = pytest.mark.requires_rust_extension STREAM: Final = ResponseSpec(body=None, events=MESSAGES_EVENTS) +@pytest.fixture(autouse=True) +def opt_messages_into_rust() -> Iterator[None]: + with rebound(catalog, "RULES", (RouteRule(Route.MESSAGES, Rollout.RUST_OPT_IN), *catalog.RULES)): + yield + + @pytest.fixture def messages_server(recording_server: RecordingServer) -> RecordingServer: recording_server.default_response = ResponseSpec(body=MESSAGES_RESPONSE) diff --git a/tests/test_litellm_rust/ocr/test_requests.py b/tests/test_litellm_rust/ocr/test_requests.py index 51815651eb4..0d3b8ba472d 100644 --- a/tests/test_litellm_rust/ocr/test_requests.py +++ b/tests/test_litellm_rust/ocr/test_requests.py @@ -593,7 +593,7 @@ def test_native_projection_errors_never_select_python( import ssl from litellm.rust_bridge import runtime, settings - from litellm.rust_bridge.catalog import Context, Route, Rule + from litellm.rust_bridge.catalog import Route, RouteContext, RouteRule from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.ocr.entrypoints import NATIVE_OCR, LiteLLMOcrRequest @@ -621,11 +621,11 @@ def test_native_projection_errors_never_select_python( with pytest.raises(RuntimeError if failure == "schema" else ValueError, match="http_settings"): runtime.run( - Context(Route.OCR, provider="mistral"), + RouteContext(Route.OCR, provider="mistral"), binding=NATIVE_OCR, native=lambda native: native(request, (), {}), python=python_fallback, - rules=(Rule(Route.OCR, Rollout.RUST_REQUIRED if required else Rollout.RUST_OPT_OUT),), + rules=(RouteRule(Route.OCR, Rollout.RUST_REQUIRED if required else Rollout.RUST_OPT_OUT),), ) assert ocr_server.requests == [] diff --git a/tests/test_litellm_rust/test_cache.py b/tests/test_litellm_rust/test_cache.py index c4962b533c4..e2e2f9f1819 100644 --- a/tests/test_litellm_rust/test_cache.py +++ b/tests/test_litellm_rust/test_cache.py @@ -15,7 +15,7 @@ from contextlib import ExitStack from datetime import datetime from pathlib import Path from types import SimpleNamespace -from typing import Final, Protocol, cast +from typing import Final, Protocol, TypeAlias, cast from unittest.mock import Mock from urllib.parse import urlparse from uuid import uuid4 @@ -37,7 +37,10 @@ from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cluster_cache import RedisClusterCache from litellm.caching.redis_semantic_cache import RedisSemanticCache from litellm.caching.s3_cache import S3Cache -from litellm.rust_bridge import _native +from litellm.rust_bridge import _native, catalog +from litellm.rust_bridge.catalog import CacheRule, Route, RouteRule, SecretManagerRule +from litellm.rust_bridge.configuration import Rollout +from litellm.rust_bridge.response_cache import NativeResponseCacheRuntime, ResponseCacheRuntime, resolve_response_cache from litellm.types.caching import LiteLLMCacheType from litellm.types.llms.custom_llm import CustomLLMItem from litellm.types.utils import EmbeddingResponse @@ -189,6 +192,7 @@ def test_existing_constructor_and_global_are_unchanged() -> None: facade: Final = Cache(type=LiteLLMCacheType.LOCAL) assert type(facade.cache) is InMemoryCache assert "_native_cache_handle" not in vars(facade) + assert resolve_response_cache(facade) is None with rebound(litellm, "cache", facade): resolver: Final = _CacheTestResolver(litellm) assert resolver.resolve().kind == "python_callback" @@ -196,6 +200,42 @@ def test_existing_constructor_and_global_are_unchanged() -> None: assert cast(CacheLookup, facade).get_cache(cache_key="key") == {"answer": 7} +async def test_catalog_constructs_native_runtime_from_public_cache_configuration() -> None: + rules: Final = ( + RouteRule(Route.OCR, Rollout.PYTHON_ONLY), + SecretManagerRule(Rollout.PYTHON_ONLY, systems=frozenset({"local"})), + CacheRule(Rollout.RUST_REQUIRED, backends=frozenset({"local"})), + ) + facade: Final = Cache(type=LiteLLMCacheType.LOCAL) + runtime: Final = resolve_response_cache(facade, rules) + assert isinstance(runtime, ResponseCacheRuntime) + assert runtime.kind == "native" + + sync_request: Final = runtime.request(facade, {"cache_key": "sync"}) + assert sync_request is not None + runtime.store(sync_request, {"answer": 1}) + assert runtime.lookup(sync_request) == {"answer": 1} + assert facade.cache.get_cache("sync") is None + + async_request: Final = runtime.request(facade, {"cache_key": "async"}) + assert async_request is not None + await runtime.async_store(async_request, {"answer": 2}) + assert await runtime.async_lookup(async_request) == {"answer": 2} + assert await facade.cache.async_get_cache("async") is None + + requests: Final = (sync_request, async_request) + expected: Final = { + "values": [{"answer": 1}, {"answer": 2}], + "missing_indices": [], + } + assert runtime.lookup_batch(requests) == expected + assert await runtime.async_lookup_batch(requests) == expected + + await runtime.async_flush() + assert runtime.lookup(sync_request) is None + assert await runtime.async_lookup(async_request) is None + + def test_existing_global_lifecycle_remains_the_resolver_source_of_truth() -> None: resolver: Final = _CacheTestResolver(litellm) @@ -410,9 +450,7 @@ async def test_memory_size_policy_is_applied_by_the_native_host() -> None: await binding.async_store(request("large"), {"answer": "x" * 256}) assert binding.lookup(request("large")) is None assert binding.lookup(request("small")) == small - disabled: Final = _CacheTestResolver( - SimpleNamespace(cache=_CacheTestHandle.memory(capacity=0)) - ).resolve() + disabled: Final = _CacheTestResolver(SimpleNamespace(cache=_CacheTestHandle.memory(capacity=0))).resolve() await disabled.async_store(request(), small) assert await disabled.async_lookup(request()) is None @@ -456,9 +494,7 @@ async def test_python_batch_callbacks_use_the_builtin_cache_api() -> None: ) -> object: return result, kwargs - binding: Final = _CacheTestResolver( - SimpleNamespace(cache=CustomCache(type=LiteLLMCacheType.LOCAL)) - ).resolve() + binding: Final = _CacheTestResolver(SimpleNamespace(cache=CustomCache(type=LiteLLMCacheType.LOCAL))).resolve() assert binding.kind == "python_callback" requests: Final = [request("first"), request("second")] kwargs: Final = [{"cache_key": "first"}, {"cache_key": "second"}] @@ -1212,10 +1248,7 @@ def _semantic_embedding(prompt: str) -> list[float]: base: Final = _base_embedding(prompt.replace(PARAPHRASE_MARKER, "").strip()) pivot: Final = min(range(8), key=lambda index: abs(base[index])) direction: Final = _normalized( - [ - (1.0 - base[pivot] * base[pivot]) if index == pivot else -base[index] * base[pivot] - for index in range(8) - ] + [(1.0 - base[pivot] * base[pivot]) if index == pivot else -base[index] * base[pivot] for index in range(8)] ) # Rotating an orthogonal unit direction by 0.329 produces ~0.05 cosine distance return _normalized([base[index] + 0.329 * direction[index] for index in range(8)]) @@ -1311,9 +1344,7 @@ def semantic_embedding() -> Generator[DeterministicEmbedding]: [*litellm._custom_providers, "semantic-test"], # pyright: ignore[reportPrivateUsage] # no public provider-registration hook ) ) - stack.enter_context( - rebound(litellm, "provider_list", [*litellm.provider_list, "semantic-test"]) - ) + stack.enter_context(rebound(litellm, "provider_list", [*litellm.provider_list, "semantic-test"])) yield handler @@ -1441,9 +1472,7 @@ async def test_redis_semantic_async_paths_and_store_batch_share_one_layout( binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve() client: Final = redis.Redis.from_url(url) - await binding.async_store( - semantic_request("async", "name a primary color"), {"answer": "blue"} - ) + await binding.async_store(semantic_request("async", "name a primary color"), {"answer": "blue"}) hash_key: Final = f"{index}:{semantic_entry_id('name a primary color', 'async')}" decoded: Final = cast(dict[str, object], json.loads(cast(bytes, client.hget(hash_key, "response")))) python_read: Final = await cast(RedisSemanticCache, facade.cache).async_get_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs stays unknown on the backend class @@ -1459,9 +1488,7 @@ async def test_redis_semantic_async_paths_and_store_batch_share_one_layout( [{"answer": 1}, {"answer": 2}], ) expected: Final = { - key: json.loads( - cast(bytes, client.hget(f"{index}:{semantic_entry_id(prompt, key)}", "response")) - ) + key: json.loads(cast(bytes, client.hget(f"{index}:{semantic_entry_id(prompt, key)}", "response"))) for key, prompt in ( ("batch-one", "first batch prompt"), ("batch-two", "second batch prompt"), @@ -1471,18 +1498,19 @@ async def test_redis_semantic_async_paths_and_store_batch_share_one_layout( ("batch-one", "first batch prompt"), ("batch-two", "second batch prompt"), ): - assert cast(RedisSemanticCache, facade.cache).get_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs stays unknown on the backend class - key, messages=semantic_messages(prompt) - ) == expected[key], key + assert ( + cast(RedisSemanticCache, facade.cache).get_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs stays unknown on the backend class + key, messages=semantic_messages(prompt) + ) + == expected[key] + ), key cast(RedisSemanticCache, facade.cache).set_cache( # pyright: ignore[reportUnknownMemberType] # **kwargs stays unknown on the backend class "async-python", json.dumps({"timestamp": 1700000000.0, "response": {"answer": "python"}}), messages=semantic_messages("python written prompt"), ) - assert await binding.async_lookup( - semantic_request("async-python", "python written prompt") - ) == {"answer": "python"} + assert await binding.async_lookup(semantic_request("async-python", "python written prompt")) == {"answer": "python"} client.close() @@ -1497,13 +1525,9 @@ async def test_native_semantic_async_embedding_runs_inline_in_the_callers_task( SEMANTIC_CONTEXT.set("caller-sentinel") response: Final = {"choices": [{"text": "paris"}]} - await binding.async_store( - semantic_request("inline", "what is the capital of france"), response - ) + await binding.async_store(semantic_request("inline", "what is the capital of france"), response) assert ( - await binding.async_lookup( - semantic_request("inline", f"what is the capital of france{PARAPHRASE_MARKER}") - ) + await binding.async_lookup(semantic_request("inline", f"what is the capital of france{PARAPHRASE_MARKER}")) == response ) assert await binding.async_lookup(semantic_request("inline", "python written prompt")) is None @@ -1540,9 +1564,7 @@ async def test_native_semantic_cancellation_during_embedding_skips_the_backend( semantic_embedding.gate = asyncio.Event() async def lookup() -> object: - return await binding.async_lookup( - semantic_request("cancel", "cancelled prompt") - ) + return await binding.async_lookup(semantic_request("cancel", "cancelled prompt")) task: Final = asyncio.create_task(lookup()) await semantic_embedding.entered.wait() @@ -1587,9 +1609,7 @@ def test_redis_semantic_ttl_is_written_only_when_requested( binding: Final = _CacheTestResolver(SimpleNamespace(cache=facade)).resolve() client: Final = redis.Redis.from_url(url) - binding.store( - {**semantic_request("ttl", "ttl prompt"), "ttl_seconds": 12.0}, {"answer": 1} - ) + binding.store({**semantic_request("ttl", "ttl prompt"), "ttl_seconds": 12.0}, {"answer": 1}) expiring: Final = f"{index}:{semantic_entry_id('ttl prompt', 'ttl')}" assert 0 < client.ttl(expiring) <= 12 @@ -1737,16 +1757,13 @@ def test_redis_semantic_handle_rejects_wrong_backends( redis_semantic_cache_index_name=index, ) subclassed_facade.cache = CustomSemanticCache( # pyright: ignore[reportAttributeAccessIssue] # facade backend slot is not declared - redis_url=url, similarity_threshold=0.8, embedding_model=SEMANTIC_EMBEDDING_MODEL, index_name=index, ) with pytest.raises(TypeError): - _CacheTestHandle.redis_semantic( - subclassed_facade.cache - )._bind_facade(subclassed_facade) + _CacheTestHandle.redis_semantic(subclassed_facade.cache)._bind_facade(subclassed_facade) replacement_facade: Final = Cache( type=LiteLLMCacheType.REDIS_SEMANTIC, @@ -1770,9 +1787,7 @@ def qdrant_facade(qdrant_url: str, collection_name: str) -> Cache: ) -def test_qdrant_semantic_facade_binds_native_and_shares_entries( - qdrant_url: str, fake_embedding_endpoint: str -) -> None: +def test_qdrant_semantic_facade_binds_native_and_shares_entries(qdrant_url: str, fake_embedding_endpoint: str) -> None: del fake_embedding_endpoint messages: Final = [{"role": "user", "content": "shared prompt"}] collection: Final = f"cache_{uuid4().hex}" @@ -1803,9 +1818,7 @@ def test_qdrant_semantic_facade_binds_native_and_shares_entries( assert facade.cache.get_cache("different-key", messages=messages) is None -async def test_qdrant_semantic_async_parity( - qdrant_url: str, fake_embedding_endpoint: str -) -> None: +async def test_qdrant_semantic_async_parity(qdrant_url: str, fake_embedding_endpoint: str) -> None: del fake_embedding_endpoint messages: Final = [{"role": "user", "content": "async prompt"}] collection: Final = f"cache_{uuid4().hex}" @@ -1830,6 +1843,34 @@ async def test_qdrant_semantic_async_parity( assert python_value["response"] == {"id": "native"} +async def test_qdrant_semantic_async_store_batch_shares_entries(qdrant_url: str, fake_embedding_endpoint: str) -> None: + del fake_embedding_endpoint + collection: Final = f"cache_{uuid4().hex}" + facade: Final = qdrant_facade(qdrant_url, collection) + handle: Final = _native._CacheTestHandle.qdrant_semantic( + qdrant_url, + collection_name=collection, + similarity_threshold=0.99, + vector_size=8, + ) + handle._bind_facade(facade) + binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=facade)).resolve() + entries: Final = [ + qdrant_request("batch-one", [{"role": "user", "content": "first batch prompt"}]), + qdrant_request("batch-two", [{"role": "user", "content": "second batch prompt"}]), + ] + await binding.async_store_batch(entries, [{"id": "one"}, {"id": "two"}]) + + assert binding.lookup(entries[0]) == {"id": "one"} + assert binding.lookup(entries[1]) == {"id": "two"} + assert (await facade.cache.async_get_cache("batch-one", messages=entries[0]["messages"]))["response"] == { + "id": "one" + } + assert (await facade.cache.async_get_cache("batch-two", messages=entries[1]["messages"]))["response"] == { + "id": "two" + } + + async def test_qdrant_semantic_malformed_entries_and_unsupported_operations( qdrant_url: str, fake_embedding_endpoint: str ) -> None: @@ -1873,9 +1914,7 @@ async def test_qdrant_semantic_malformed_entries_and_unsupported_operations( await binding.ping() -def test_qdrant_semantic_ignores_request_expiry( - qdrant_url: str, fake_embedding_endpoint: str -) -> None: +def test_qdrant_semantic_ignores_request_expiry(qdrant_url: str, fake_embedding_endpoint: str) -> None: del fake_embedding_endpoint messages: Final = [{"role": "user", "content": "persistent prompt"}] collection: Final = f"cache_{uuid4().hex}" @@ -1896,9 +1935,7 @@ def test_qdrant_semantic_ignores_request_expiry( assert python_value["response"] == {"id": "persistent"} -def test_qdrant_semantic_mutation_and_projection_fallback( - qdrant_url: str, fake_embedding_endpoint: str -) -> None: +def test_qdrant_semantic_mutation_and_projection_fallback(qdrant_url: str, fake_embedding_endpoint: str) -> None: del fake_embedding_endpoint collection: Final = f"cache_{uuid4().hex}" facade: Final = qdrant_facade(qdrant_url, collection) @@ -1921,3 +1958,389 @@ def test_qdrant_semantic_mutation_and_projection_fallback( unsupported.cache.qdrant_api_base = "http://127.0.0.1:7777" with pytest.raises(TypeError, match="gRPC"): handle._bind_facade(unsupported) + + +CacheFactory: TypeAlias = Callable[[], Cache] + + +def require_rust(monkeypatch: pytest.MonkeyPatch, backend: LiteLLMCacheType) -> None: + monkeypatch.setattr(catalog, "RULES", (CacheRule(Rollout.RUST_REQUIRED, backends=frozenset({backend})),)) + + +def native_runtime(facade: Cache) -> ResponseCacheRuntime: + runtime: Final = facade._native_cache # pyright: ignore[reportPrivateUsage] # the activation under test has no public accessor + assert isinstance(runtime, ResponseCacheRuntime) + assert runtime.kind == "native" + return runtime + + +@pytest.fixture +def cache_factory(request: pytest.FixtureRequest, tmp_path: Path) -> CacheFactory: + backend: Final = cast(LiteLLMCacheType, request.param) + match backend: + case LiteLLMCacheType.LOCAL: + return lambda: Cache(type=backend) + case LiteLLMCacheType.DISK: + return lambda: Cache(type=backend, disk_cache_dir=str(tmp_path)) + case LiteLLMCacheType.REDIS: + parsed: Final = urlparse(cast(str, request.getfixturevalue("redis_url"))) + return lambda: Cache(type=backend, host=parsed.hostname, port=str(parsed.port)) + case LiteLLMCacheType.S3: + stub: Final = cast(S3Stub, request.getfixturevalue("s3_stub")) + return lambda: Cache( + type=backend, + s3_bucket_name="cache-bucket", + s3_region_name="us-east-1", + s3_endpoint_url=stub.url, + s3_aws_access_key_id="key", + s3_aws_secret_access_key="secret", + s3_path="team", + ) + case LiteLLMCacheType.GCS: + return lambda: Cache(type=backend, gcs_bucket_name="bucket", gcs_path="cache/") + case LiteLLMCacheType.REDIS_SEMANTIC: + return lambda: Cache( + type=backend, + redis_url="redis://127.0.0.1:6379", + similarity_threshold=0.8, + redis_semantic_cache_embedding_model="text-embedding-3-small", + ) + case LiteLLMCacheType.VALKEY_SEMANTIC: + return lambda: Cache(type=backend, redis_url="redis://127.0.0.1:6390/0", similarity_threshold=0.8) + case _: + raise AssertionError(f"no local factory for {backend}") + + +ROUND_TRIP_BACKENDS: Final = ( + LiteLLMCacheType.LOCAL, + LiteLLMCacheType.DISK, + LiteLLMCacheType.REDIS, + LiteLLMCacheType.S3, +) +SHARED_STORE_BACKENDS: Final = (LiteLLMCacheType.DISK, LiteLLMCacheType.REDIS, LiteLLMCacheType.S3) + + +def completion_kwargs(label: str) -> dict[str, object]: + return {"model": "gpt-4o", "messages": [{"role": "user", "content": f"{label} {uuid4().hex}"}]} + + +@pytest.mark.parametrize("backend", list(LiteLLMCacheType)) +def test_shipped_rules_keep_every_backend_on_python(backend: LiteLLMCacheType) -> None: + assert resolve_response_cache(cast(Cache, SimpleNamespace(type=backend))) is None + + +@pytest.mark.parametrize( + "cache_factory", + [ + LiteLLMCacheType.LOCAL, + LiteLLMCacheType.DISK, + LiteLLMCacheType.REDIS, + LiteLLMCacheType.S3, + LiteLLMCacheType.GCS, + LiteLLMCacheType.REDIS_SEMANTIC, + LiteLLMCacheType.VALKEY_SEMANTIC, + ], + indirect=True, +) +def test_shipped_rules_construct_python_backed_facades(cache_factory: CacheFactory) -> None: + assert cache_factory()._native_cache is None # pyright: ignore[reportPrivateUsage] # the activation under test has no public accessor + + +@pytest.mark.parametrize( + "cache_factory", + [ + LiteLLMCacheType.LOCAL, + LiteLLMCacheType.DISK, + LiteLLMCacheType.REDIS, + LiteLLMCacheType.S3, + LiteLLMCacheType.GCS, + LiteLLMCacheType.REDIS_SEMANTIC, + LiteLLMCacheType.VALKEY_SEMANTIC, + ], + indirect=True, +) +def test_rust_required_rule_activates_the_native_backend( + cache_factory: CacheFactory, monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest +) -> None: + require_rust(monkeypatch, cast(LiteLLMCacheType, request.node.callspec.params["cache_factory"])) + native_runtime(cache_factory()) + + +@pytest.mark.parametrize("cache_factory", ROUND_TRIP_BACKENDS, indirect=True) +async def test_facade_storage_calls_round_trip_through_the_native_backend( + cache_factory: CacheFactory, monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest +) -> None: + require_rust(monkeypatch, cast(LiteLLMCacheType, request.node.callspec.params["cache_factory"])) + facade: Final = cache_factory() + native_runtime(facade) + + sync_kwargs: Final = completion_kwargs("sync") + facade.add_cache({"answer": 1}, **sync_kwargs) + assert facade.get_cache(**sync_kwargs) == {"answer": 1} + + async_kwargs: Final = completion_kwargs("async") + await facade.async_add_cache({"answer": 2}, **async_kwargs) + assert await facade.async_get_cache(**async_kwargs) == {"answer": 2} + assert facade.get_cache(**completion_kwargs("absent")) is None + + +async def test_memory_facade_writes_bypass_the_python_backend(monkeypatch: pytest.MonkeyPatch) -> None: + require_rust(monkeypatch, LiteLLMCacheType.LOCAL) + facade: Final = Cache(type=LiteLLMCacheType.LOCAL) + native_runtime(facade) + kwargs: Final = completion_kwargs("memory") + facade.add_cache({"answer": 1}, **kwargs) + assert facade.cache.get_cache(facade.get_cache_key(**kwargs)) is None + assert facade.get_cache(**kwargs) == {"answer": 1} + + +@pytest.mark.parametrize("cache_factory", SHARED_STORE_BACKENDS, indirect=True) +async def test_native_and_python_facades_share_one_wire_format( + cache_factory: CacheFactory, monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest +) -> None: + python_facade: Final = cache_factory() + assert python_facade._native_cache is None # pyright: ignore[reportPrivateUsage] # the activation under test has no public accessor + require_rust(monkeypatch, cast(LiteLLMCacheType, request.node.callspec.params["cache_factory"])) + native_facade: Final = cache_factory() + native_runtime(native_facade) + + native_written: Final = completion_kwargs("native") + native_facade.add_cache({"writer": "native"}, **native_written) + assert python_facade.get_cache(**native_written) == {"writer": "native"} + + python_written: Final = completion_kwargs("python") + python_facade.add_cache({"writer": "python"}, **python_written) + assert native_facade.get_cache(**python_written) == {"writer": "python"} + + async_native: Final = completion_kwargs("async-native") + await native_facade.async_add_cache({"writer": "async-native"}, **async_native) + assert await python_facade.async_get_cache(**async_native) == {"writer": "async-native"} + + async_python: Final = completion_kwargs("async-python") + await python_facade.async_add_cache({"writer": "async-python"}, **async_python) + assert await native_facade.async_get_cache(**async_python) == {"writer": "async-python"} + + +@pytest.mark.parametrize("cache_factory", ROUND_TRIP_BACKENDS, indirect=True) +async def test_embedding_pipeline_stores_one_native_entry_per_input( + cache_factory: CacheFactory, monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest +) -> None: + require_rust(monkeypatch, cast(LiteLLMCacheType, request.node.callspec.params["cache_factory"])) + facade: Final = cache_factory() + native_runtime(facade) + inputs: Final = [f"alpha {uuid4().hex}", f"beta {uuid4().hex}"] + result: Final = EmbeddingResponse( + model="text-embedding-3-small", + data=[ + {"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}, + {"object": "embedding", "index": 1, "embedding": [0.3, 0.4]}, + ], + ) + await facade.async_add_cache_pipeline(result, model="text-embedding-3-small", input=inputs) + + keys: Final = [facade.get_cache_key(model="text-embedding-3-small", input=text) for text in inputs] + assert len(set(keys)) == len(inputs) + for text, expected in zip(inputs, ([0.1, 0.2], [0.3, 0.4]), strict=True): + cached = await facade.async_get_cache(model="text-embedding-3-small", input=text) + assert isinstance(cached, dict) + assert cached["embedding"] == expected + assert await facade.async_get_cache(model="text-embedding-3-small", input=inputs) is None + + +def redis_facade(redis_url: str, **settings: object) -> Cache: + parsed: Final = urlparse(redis_url) + return Cache(type=LiteLLMCacheType.REDIS, host=parsed.hostname, port=str(parsed.port), **settings) + + +@pytest.mark.parametrize( + ("settings", "message"), + [ + pytest.param({"max_connections": 10}, "max_connections requires Python", id="pool-size"), + pytest.param({"socket_timeout": 1.0}, "socket_timeout and socket_connect_timeout", id="socket-timeout"), + pytest.param( + {"socket_connect_timeout": 1.0}, "socket_timeout and socket_connect_timeout", id="connect-timeout" + ), + pytest.param({"socket_keepalive": True}, "does not support socket_keepalive", id="keepalive"), + pytest.param({"health_check_interval": 5}, "does not support health_check_interval", id="health-check"), + pytest.param({"client_name": "litellm"}, "does not support client_name", id="client-name"), + pytest.param({"ssl": True}, "ssl_check_hostname=false require Python", id="tls-default-hostname-check"), + pytest.param({"ssl": True, "ssl_cert_reqs": "none"}, "ssl_cert_reqs=none", id="tls-without-verification"), + pytest.param( + {"ssl": True, "ssl_check_hostname": True, "ssl_ca_certs": "/ca.pem"}, + "does not support ssl_ca_certs", + id="tls-custom-ca", + ), + pytest.param( + {"ssl": True, "ssl_check_hostname": True, "ssl_certfile": "/client.pem", "ssl_keyfile": "/client.key"}, + "does not support ssl_ca_certs, ssl_ca_data, ssl_certfile or ssl_keyfile", + id="tls-client-certificate", + ), + ], +) +def test_redis_settings_the_native_client_cannot_honor_decline( + redis_url: str, monkeypatch: pytest.MonkeyPatch, settings: dict[str, object], message: str +) -> None: + require_rust(monkeypatch, LiteLLMCacheType.REDIS) + with pytest.raises(RuntimeError, match=f"declined the cache: native Redis.*{message}"): + redis_facade(redis_url, **settings) + + +def test_redis_verified_tls_activates_natively(redis_url: str, monkeypatch: pytest.MonkeyPatch) -> None: + require_rust(monkeypatch, LiteLLMCacheType.REDIS) + native_runtime(redis_facade(redis_url, ssl=True, ssl_check_hostname=True)) + + +async def test_redis_flush_size_buffers_native_facade_writes(redis_url: str, monkeypatch: pytest.MonkeyPatch) -> None: + require_rust(monkeypatch, LiteLLMCacheType.REDIS) + facade: Final = redis_facade(redis_url, redis_flush_size=2, namespace="team") + native_runtime(facade) + client: Final = redis.Redis.from_url(redis_url) + first: Final = completion_kwargs("first") + await facade.async_add_cache({"value": 1}, **first) + first_key: Final = facade.get_cache_key(**first) + assert first_key.startswith("team:") + assert client.get(first_key) is None + second: Final = completion_kwargs("second") + await facade.async_add_cache({"value": 2}, **second) + assert client.get(first_key) is not None + assert client.get(facade.get_cache_key(**second)) is not None + client.close() + + +@pytest.mark.parametrize( + ("backend", "settings", "message"), + [ + pytest.param( + LiteLLMCacheType.VALKEY_SEMANTIC, + {"redis_url": "rediss://127.0.0.1:6390/0", "similarity_threshold": 0.8}, + "native Valkey semantic cache does not support TLS connections", + id="valkey-tls", + ), + pytest.param( + LiteLLMCacheType.VALKEY_SEMANTIC, + {"redis_url": "redis://127.0.0.1:6390/0?socket_timeout=1", "similarity_threshold": 0.8}, + "native Redis uses fixed socket timeouts; socket_timeout and socket_connect_timeout require Python", + id="valkey-socket-timeout", + ), + pytest.param( + LiteLLMCacheType.REDIS_SEMANTIC, + {"redis_url": "rediss://127.0.0.1:6380", "similarity_threshold": 0.8}, + "native Redis semantic cache does not support TLS or query options in redis_url", + id="redis-semantic-tls", + ), + pytest.param( + LiteLLMCacheType.REDIS_SEMANTIC, + {"redis_url": "redis://127.0.0.1:6379?socket_timeout=1", "similarity_threshold": 0.8}, + "native Redis semantic cache does not support TLS or query options in redis_url", + id="redis-semantic-query", + ), + ], +) +def test_semantic_settings_the_native_client_cannot_honor_decline( + monkeypatch: pytest.MonkeyPatch, backend: LiteLLMCacheType, settings: dict[str, object], message: str +) -> None: + require_rust(monkeypatch, backend) + with pytest.raises(RuntimeError, match=f"declined the cache: {message}"): + Cache(type=backend, **settings) + + +def test_rust_with_fallback_keeps_python_when_the_native_client_declines( + redis_url: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + catalog, + "RULES", + (CacheRule(Rollout.RUST_OPT_OUT, backends=frozenset({LiteLLMCacheType.REDIS})),), + ) + assert redis_facade(redis_url, socket_timeout=1.0)._native_cache is None # pyright: ignore[reportPrivateUsage] # the activation under test has no public accessor + + +def test_qdrant_semantic_rust_required_rule_activates_natively( + qdrant_url: str, fake_embedding_endpoint: str, monkeypatch: pytest.MonkeyPatch +) -> None: + del fake_embedding_endpoint + require_rust(monkeypatch, LiteLLMCacheType.QDRANT_SEMANTIC) + facade: Final = qdrant_facade(qdrant_url, f"cache_{uuid4().hex}") + native_runtime(facade) + kwargs: Final = {"model": "gpt-4o", "messages": [{"role": "user", "content": "qdrant activation"}]} + facade.add_cache({"answer": "qdrant"}, **kwargs) + assert facade.get_cache(**kwargs) == {"answer": "qdrant"} + + +async def test_redis_semantic_rust_required_rule_activates_natively( + redis_stack: tuple[str, str], semantic_embedding: DeterministicEmbedding, monkeypatch: pytest.MonkeyPatch +) -> None: + del semantic_embedding + url, index = redis_stack + require_rust(monkeypatch, LiteLLMCacheType.REDIS_SEMANTIC) + facade: Final = Cache( + type=LiteLLMCacheType.REDIS_SEMANTIC, + redis_url=url, + similarity_threshold=0.8, + redis_semantic_cache_embedding_model=SEMANTIC_EMBEDDING_MODEL, + redis_semantic_cache_index_name=index, + ) + native_runtime(facade) + kwargs: Final = {"model": "gpt-4o", "messages": semantic_messages("name a primary color")} + await facade.async_add_cache({"answer": "blue"}, **kwargs) + assert await facade.async_get_cache(**kwargs) == {"answer": "blue"} + + +async def test_azure_blob_rust_required_rule_activates_natively(monkeypatch: pytest.MonkeyPatch) -> None: + account_url: Final = os.environ.get("AZURE_BLOB_CACHE_ACCOUNT_URL") + if account_url is None: + pytest.skip( + "live Azure Blob parity needs AZURE_BLOB_CACHE_ACCOUNT_URL plus DefaultAzureCredential inputs in the environment" + ) + require_rust(monkeypatch, LiteLLMCacheType.AZURE_BLOB) + facade: Final = Cache( + type=LiteLLMCacheType.AZURE_BLOB, + azure_account_url=account_url, + azure_blob_container=f"litellm-parity-{uuid.uuid4().hex[:12]}", + ) + backend: Final = facade.cache + assert isinstance(backend, AzureBlobCache) + try: + native_runtime(facade) + kwargs: Final = completion_kwargs("azure") + await facade.async_add_cache({"answer": "azure"}, **kwargs) + assert await facade.async_get_cache(**kwargs) == {"answer": "azure"} + assert backend.get_cache(facade.get_cache_key(**kwargs))["response"] == {"answer": "azure"} + finally: + backend.container_client.delete_container() + await backend.disconnect() + + +class _SemanticHit: + """A native semantic runtime that answers every lookup with one cached response.""" + + kind: Final = "native" + + def lookup_semantic(self, request: object) -> tuple[object, float | None]: + return {"answer": 42}, 0.97 + + async def async_lookup_semantic(self, request: object) -> tuple[object, float | None]: + return {"answer": 42}, 0.97 + + +@pytest.mark.parametrize("semantic_type", [LiteLLMCacheType.QDRANT_SEMANTIC, LiteLLMCacheType.REDIS_SEMANTIC]) +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +def test_native_semantic_hit_stamps_similarity_on_request_metadata( + semantic_type: LiteLLMCacheType, use_async: bool +) -> None: + """Python semantic backends write `metadata["semantic-similarity"]` on every lookup, and the + facade copies it to the caller's metadata; the native path must report it the same way.""" + facade: Final = Cache() + facade.type = semantic_type + facade._native_cache = ResponseCacheRuntime(cast(NativeResponseCacheRuntime, _SemanticHit())) # pyright: ignore[reportPrivateUsage] # the native path under test has no public setter + metadata: Final[dict[str, object]] = {} + kwargs: Final = { + "cache_key": "semantic-key", + "messages": [{"role": "user", "content": "hello"}], + "metadata": metadata, + } + + result: Final = asyncio.run(facade.async_get_cache(**kwargs)) if use_async else facade.get_cache(**kwargs) + + assert result == {"answer": 42} + assert metadata["semantic-similarity"] == 0.97 diff --git a/tests/test_litellm_rust/test_fork_guard.py b/tests/test_litellm_rust/test_fork_guard.py index 086397bab5c..02c454e0460 100644 --- a/tests/test_litellm_rust/test_fork_guard.py +++ b/tests/test_litellm_rust/test_fork_guard.py @@ -1,5 +1,6 @@ import os import textwrap +from typing import Final import pytest @@ -144,3 +145,83 @@ def test_sdk_call_in_a_child_forked_after_native_use_raises_instead_of_hanging() result = run_child_interpreter(_SDK_CONTRACT, env=env, timeout=120) assert result.returncode == 0, result.stderr + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="fork only") +@pytest.mark.parametrize("warm_fast_counter", (False, True)) +def test_tokenizers_share_the_native_process_guard(warm_fast_counter: bool) -> None: + script: Final = """ +import asyncio +import os +import litellm +from litellm.proxy.spend_tracking.input_tokens import count_input_tokens +from litellm.rust_bridge import _native, catalog +from litellm.rust_bridge.catalog import Route, RouteRule +from litellm.rust_bridge.configuration import Rollout +from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer +from litellm.utils import claude_json_str + +catalog.RULES = ( + RouteRule(Route.TOKENIZER, Rollout.RUST_OPT_IN), + RouteRule(Route.TOKEN_COUNTER, Rollout.RUST_OPT_IN), + *catalog.RULES, +) +litellm.anthropic_models = {*litellm.anthropic_models, "tokenizer-fork-fixture"} +_native.reserve_process_for_forking() +for create in ( + lambda: _native.Tokenizer.from_tiktoken("cl100k_base"), + lambda: _native.Tokenizer.from_json(claude_json_str), + lambda: litellm.token_counter(model="tokenizer-fork-fixture", text="hello"), +): + try: + create() + except _native.ProcessReservedForForking: + pass + else: + raise AssertionError("reserved parent ran a native tokenizer") +assert not _native.process_state_started() + +pid = os.fork() +if pid == 0: + tokenizer = HuggingFaceTokenizer.from_str(claude_json_str) + encoding = _native.Tokenizer.from_tiktoken("cl100k_base") + if os.environ["WARM_FAST_COUNTER"] == "True": + _native.TokenCounter.from_tokenizer(encoding, fast=True) + expected = [item.ids for item in tokenizer.encode_batch(["hello", "world"])] + assert _native.process_state_started() + grandchild = os.fork() + if grandchild == 0: + for call in ( + lambda: tokenizer.encode_batch(["hello", "world"]), + lambda: tokenizer.encode("hello"), + lambda: encoding.count("hello"), + lambda: encoding.count("hello", fast=True), + lambda: _native.TokenCounter.from_tokenizer(encoding), + lambda: _native.TokenCounter.from_tokenizer(encoding, fast=True), + lambda: _native.Tokenizer.from_tiktoken("cl100k_base"), + lambda: asyncio.run(count_input_tokens({"prompt": "hello"}, b'{"prompt": "hello"}', ("counter-fork-fixture",))), + ): + try: + call() + except _native.ForkedAfterNativeRuntimeStarted: + pass + else: + os._exit(1) + os._exit(0) + assert os.waitpid(grandchild, 0)[1] == 0 + assert [item.ids for item in tokenizer.encode_batch(["hello", "world"])] == expected + os._exit(0) +assert os.waitpid(pid, 0)[1] == 0 +""" + result: Final = run_child_interpreter( + script, + env={ + **os.environ, + "OBJC_DISABLE_INITIALIZE_FORK_SAFETY": "YES", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + "WARM_FAST_COUNTER": str(warm_fast_counter), + }, + timeout=30, + ) + + assert result.returncode == 0, result.stderr diff --git a/tests/test_litellm_rust/test_tokenizer.py b/tests/test_litellm_rust/test_tokenizer.py new file mode 100644 index 00000000000..98d5259b652 --- /dev/null +++ b/tests/test_litellm_rust/test_tokenizer.py @@ -0,0 +1,130 @@ +import json +from typing import Final + +import pytest +import tiktoken +from tokenizers import Tokenizer as ReferenceTokenizer + +from litellm.rust_bridge import _native +from litellm.utils import claude_json_str +from tests.test_litellm.litellm_core_utils.test_decode_special_tokens import TOKENIZER_JSON + +pytestmark = pytest.mark.requires_rust_extension + + +def test_tiktoken_codec_round_trips_and_counts() -> None: + tokenizer: Final = _native.Tokenizer.from_tiktoken("cl100k_base") + encoded: Final = tokenizer.encode("hello world") + + assert tokenizer.name == "cl100k_base" + assert tokenizer.count("hello world") == len(encoded) + assert tokenizer.decode(encoded) == "hello world" + + +def test_huggingface_codec_skips_special_tokens() -> None: + tokenizer: Final = _native.Tokenizer.from_json(claude_json_str) + encoded: Final = tokenizer.encode("hello") + + assert "" in tokenizer.decode(encoded, skip_special_tokens=False) + assert tokenizer.decode(encoded, skip_special_tokens=True) == "hello" + + +def test_tiktoken_codec_keeps_the_requested_encoding_name() -> None: + assert _native.Tokenizer.from_tiktoken("gpt2").name == "gpt2" + assert _native.Tokenizer.from_tiktoken("r50k_base").name == "r50k_base" + assert _native.Tokenizer.from_tiktoken("gpt2").encode("hi") == _native.Tokenizer.from_tiktoken("r50k_base").encode( + "hi" + ) + + +def test_tiktoken_codec_exposes_its_vocabulary() -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + tokenizer: Final = _native.Tokenizer.from_tiktoken("cl100k_base") + + assert tokenizer.special_tokens() == reference._special_tokens + assert tokenizer.max_token_value() == reference.max_token_value + assert tokenizer.token_byte_values() == reference.token_byte_values() + assert tokenizer.encode_single_token(b"hello") == reference.encode_single_token("hello") + assert tokenizer.is_special_token(reference.eot_token) and not tokenizer.is_special_token(0) + with pytest.raises(KeyError): + tokenizer.encode_single_token(b"<|not-a-token|>") + + +def test_huggingface_codec_rejects_tiktoken_only_calls() -> None: + tokenizer: Final = _native.Tokenizer.from_json(claude_json_str) + with pytest.raises(ValueError, match="requires a tiktoken encoding"): + tokenizer.token_byte_values() + with pytest.raises(ValueError, match="requires a Hugging Face tokenizer"): + _native.Tokenizer.from_tiktoken("cl100k_base").get_vocab() + + +def test_unknown_tiktoken_encoding_raises_value_error() -> None: + with pytest.raises(ValueError, match="unsupported tokenizer"): + _native.Tokenizer.from_tiktoken("unknown-encoding") + + +def test_tiktoken_codec_decodes_truncated_unicode_like_python() -> None: + reference: Final = tiktoken.get_encoding("cl100k_base") + tokenizer: Final = _native.Tokenizer.from_tiktoken(reference.name) + encoded: Final = reference.encode("🙂漢字") + + assert tuple(tokenizer.decode(encoded[:end]) for end in range(1, len(encoded) + 1)) == tuple( + reference.decode(encoded[:end]) for end in range(1, len(encoded) + 1) + ) + + +FAST_TEXTS: Final = ( + "", + "hello world <|endoftext|>", + "café 漢字 ع 🙂 line\r\n indented 123456789", + "x a\u0301 fi", +) + + +def test_fast_counting_is_an_opt_in_over_the_same_loaded_tokenizer() -> None: + for tokenizer in ( + _native.Tokenizer.from_tiktoken("cl100k_base"), + _native.Tokenizer.from_tiktoken("o200k_base"), + _native.Tokenizer.from_json(claude_json_str), + ): + assert [tokenizer.count(text, fast=True) for text in FAST_TEXTS] == [ + tokenizer.count(text) for text in FAST_TEXTS + ] + + +@pytest.mark.parametrize( + "name", ("cl100k_base", "o200k_base", "o200k_harmony", "p50k_base", "p50k_edit", "r50k_base", "gpt2") +) +@pytest.mark.asyncio +async def test_token_counter_counts_over_a_shared_tokenizer(name: str) -> None: + messages: Final = [{"role": "user", "content": "hello wide world"}, {"role": "assistant", "content": "ok"}] + body: Final = json.dumps({"model": "gpt-4", "messages": messages}).encode() + tokenizer: Final = _native.Tokenizer.from_tiktoken(name) + reference: Final = tiktoken.get_encoding(name) + for text in FAST_TEXTS: + assert tokenizer.count(text, fast=True) == tokenizer.count(text) == len(reference.encode_ordinary(text)) + + exact: Final = await _native.TokenCounter.from_tokenizer(tokenizer).acount_request(body) + fast: Final = await _native.TokenCounter.from_tokenizer(tokenizer, fast=True).acount_request(body) + + assert exact == fast + assert exact["input_tokens"] == 3 + sum( + 3 + len(reference.encode_ordinary(message["role"])) + len(reference.encode_ordinary(message["content"])) + for message in messages + ) + + +@pytest.mark.parametrize("configured", (False, True)) +@pytest.mark.asyncio +async def test_fast_count_preserves_huggingface_configuration(configured: bool) -> None: + reference: Final = ReferenceTokenizer.from_str(TOKENIZER_JSON) + if configured: + reference.enable_truncation(max_length=3) + reference.enable_padding(pad_id=0, pad_token="[UNK]", length=5) + tokenizer: Final = _native.Tokenizer.from_json(reference.to_str()) + counter: Final = _native.TokenCounter.from_tokenizer(tokenizer, fast=True) + for text in ("", "Hello", "Hello World Hello World", "[BOS] Hello"): + expected: Final = len(reference.encode(text)) + assert tokenizer.count(text, fast=True) == tokenizer.count(text) == expected + result: Final = await counter.acount_request(json.dumps({"prompt": text}).encode()) + assert result["input_tokens"] == expected diff --git a/tests/test_litellm_rust/test_valkey_semantic_cache_native.py b/tests/test_litellm_rust/test_valkey_semantic_cache_native.py index c87a9f86a80..046f3a70ae8 100644 --- a/tests/test_litellm_rust/test_valkey_semantic_cache_native.py +++ b/tests/test_litellm_rust/test_valkey_semantic_cache_native.py @@ -15,7 +15,10 @@ import redis from litellm.caching.caching import Cache from litellm.caching.valkey_semantic_cache import ValkeySemanticCache -from litellm.rust_bridge import _native +from litellm.rust_bridge import _native, catalog +from litellm.rust_bridge.catalog import CacheRule +from litellm.rust_bridge.configuration import Rollout +from litellm.rust_bridge.response_cache import ResponseCacheRuntime from litellm.types.caching import LiteLLMCacheType pytestmark: Final = pytest.mark.requires_rust_extension @@ -597,3 +600,22 @@ async def test_ping_maps_unsupported_native_operation_to_not_implemented( binding: Final = _native._CacheTestResolver(SimpleNamespace(cache=handle)).resolve() with pytest.raises(NotImplementedError): await binding.ping() + + +async def test_rust_required_rule_activates_the_facade_natively( + valkey_url: str, + index_name: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + catalog, + "RULES", + (CacheRule(Rollout.RUST_REQUIRED, backends=frozenset({LiteLLMCacheType.VALKEY_SEMANTIC})),), + ) + facade: Final = _facade(valkey_url, index_name, {"semantic cache prompt": [1.0, 0.0]}) + runtime: Final = facade._native_cache # pyright: ignore[reportPrivateUsage] # the activation under test has no public accessor + assert isinstance(runtime, ResponseCacheRuntime) + assert runtime.kind == "native" + kwargs: Final = {"model": "gpt-4o", "messages": _request()["messages"]} + await facade.async_add_cache({"answer": "valkey"}, **kwargs) + assert await facade.async_get_cache(**kwargs) == {"answer": "valkey"} diff --git a/tests/unified_google_tests/conftest.py b/tests/unified_google_tests/conftest.py index cd05c856faf..9eac5f49651 100644 --- a/tests/unified_google_tests/conftest.py +++ b/tests/unified_google_tests/conftest.py @@ -102,7 +102,7 @@ def google_genai_proxy_url() -> Iterator[str]: credentials_file = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", "") if not (credentials_file and os.path.isfile(credentials_file)): vertex_credentials_path = load_vertex_ai_credentials( - model="vertex_ai/gemini-2.5-flash-lite" + model="vertex_ai/gemini-3.5-flash-lite" ) if vertex_credentials_path: temp_credentials_path = vertex_credentials_path diff --git a/tests/unified_google_tests/google_genai_proxy_test_config.yaml b/tests/unified_google_tests/google_genai_proxy_test_config.yaml index 0a1779aa3ec..99f24e62916 100644 --- a/tests/unified_google_tests/google_genai_proxy_test_config.yaml +++ b/tests/unified_google_tests/google_genai_proxy_test_config.yaml @@ -1,12 +1,12 @@ model_list: - - model_name: gemini-2.5-flash-lite + - model_name: gemini-3.5-flash-lite litellm_params: - model: gemini/gemini-2.5-flash-lite + model: gemini/gemini-3.5-flash-lite api_key: os.environ/GEMINI_API_KEY - - model_name: vertex-gemini-2.5-flash-lite + - model_name: vertex-gemini-3.5-flash-lite litellm_params: - model: vertex_ai/gemini-2.5-flash-lite + model: vertex_ai/gemini-3.5-flash-lite vertex_location: global router_settings: diff --git a/tests/unified_google_tests/test_google_ai_studio.py b/tests/unified_google_tests/test_google_ai_studio.py index 6d4c3725080..2364a01cedb 100644 --- a/tests/unified_google_tests/test_google_ai_studio.py +++ b/tests/unified_google_tests/test_google_ai_studio.py @@ -13,12 +13,12 @@ class TestGoogleGenAIStudio(BaseGoogleGenAITest, BaseGoogleGenAIProxySDKTest): @property def model_config(self): return { - "model": "gemini/gemini-2.5-flash-lite", + "model": "gemini/gemini-3.5-flash-lite", } @property def proxy_model_name(self) -> str: - return "gemini-2.5-flash-lite" + return "gemini-3.5-flash-lite" @pytest.mark.asyncio @@ -94,7 +94,7 @@ async def test_mock_stream_generate_content_with_tools(): "\n--- Testing async agenerate_content_stream with function call parsing ---" ) response = await litellm.google_genai.agenerate_content_stream( - model="gemini/gemini-2.5-flash-lite", + model="gemini/gemini-3.5-flash-lite", contents=contents, tools=[ { @@ -343,7 +343,7 @@ async def test_validate_post_request_parameters(): # Make the API call response = await litellm.google_genai.agenerate_content_stream( - model="gemini/gemini-2.5-flash-lite", contents=contents, tools=tools + model="gemini/gemini-3.5-flash-lite", contents=contents, tools=tools ) # Consume the response to ensure the request is made @@ -387,11 +387,11 @@ async def test_validate_post_request_parameters(): # Validate model field assert "model" in request_data, "Expected 'model' field in request data" - # Model might be transformed, but should contain gemini-2.5-flash-lite + # Model might be transformed, but should contain gemini-3.5-flash-lite model_value = request_data["model"] assert ( - "gemini-2.5-flash-lite" in model_value - ), f"Expected model to contain 'gemini-2.5-flash-lite', got: {model_value}" + "gemini-3.5-flash-lite" in model_value + ), f"Expected model to contain 'gemini-3.5-flash-lite', got: {model_value}" print(f"✅ Model validation passed: {model_value}") # Validate contents field diff --git a/tests/unified_google_tests/test_google_genai_proxy_test_config.py b/tests/unified_google_tests/test_google_genai_proxy_test_config.py index 694ec336bac..272e589e94a 100644 --- a/tests/unified_google_tests/test_google_genai_proxy_test_config.py +++ b/tests/unified_google_tests/test_google_genai_proxy_test_config.py @@ -12,14 +12,13 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm import Router from litellm.constants import INITIAL_RETRY_DELAY, MAX_RETRY_DELAY -from litellm.llms.vertex_ai.common_utils import get_vertex_base_url +from litellm.llms.vertex_ai.common_utils import _get_gemini_url, get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase CONFIG_PATH: Final = Path(__file__).parent / "google_genai_proxy_test_config.yaml" -GEMINI_DEPLOYMENT: Final = "gemini-2.5-flash-lite" -VERTEX_DEPLOYMENT: Final = "vertex-gemini-2.5-flash-lite" -GEMINI_HOST: Final = "generativelanguage.googleapis.com" -GEMINI_GENERATE_CONTENT_PATH: Final = "/v1beta/models/gemini-2.5-flash-lite:generateContent" +GEMINI_DEPLOYMENT: Final = "gemini-3.5-flash-lite" +VERTEX_DEPLOYMENT: Final = "vertex-gemini-3.5-flash-lite" +GEMINI_GENERATE_CONTENT_URL: Final = _get_gemini_url(mode="chat", model=GEMINI_DEPLOYMENT, stream=False)[0] VERTEX_GLOBAL_BASE_URL: Final = "https://aiplatform.googleapis.com" RESOURCE_EXHAUSTED: Final = { "error": {"code": 429, "message": "Resource exhausted. Please try again later.", "status": "RESOURCE_EXHAUSTED"} @@ -81,7 +80,7 @@ async def test_ci_proxy_config_rides_out_consecutive_429s_with_backoff( ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() - route: Final = respx_mock.post(host=GEMINI_HOST, path=GEMINI_GENERATE_CONTENT_PATH).mock( + route: Final = respx_mock.post(GEMINI_GENERATE_CONTENT_URL).mock( side_effect=[httpx.Response(429, json=RESOURCE_EXHAUSTED)] * CONSECUTIVE_RATE_LIMITS + [httpx.Response(200, json=PONG)] ) diff --git a/tests/unified_google_tests/test_vertex_ai_native.py b/tests/unified_google_tests/test_vertex_ai_native.py index 640157bc33e..4bdc1a55283 100644 --- a/tests/unified_google_tests/test_vertex_ai_native.py +++ b/tests/unified_google_tests/test_vertex_ai_native.py @@ -8,9 +8,10 @@ class TestVertexAIGenerateContent(BaseGoogleGenAITest, BaseGoogleGenAIProxySDKTe @property def model_config(self): return { - "model": "vertex_ai/gemini-2.5-flash-lite", + "model": "vertex_ai/gemini-3.5-flash-lite", + "vertex_location": "global", } @property def proxy_model_name(self) -> str: - return "vertex-gemini-2.5-flash-lite" + return "vertex-gemini-3.5-flash-lite" diff --git a/tests/unit/batches/test_batch_utils.py b/tests/unit/batches/test_batch_utils.py index 708c472939e..a4de8eee23c 100644 --- a/tests/unit/batches/test_batch_utils.py +++ b/tests/unit/batches/test_batch_utils.py @@ -26,7 +26,7 @@ from openai.types.batch import BatchRequestCounts import litellm import litellm.batches.batch_utils as bu -from litellm.types.utils import LiteLLMBatch, Usage +from litellm.types.utils import LiteLLMBatch, ModelInfo, Usage # --------------------------------------------------------------------------- # # Builders for batch OUTPUT file rows. @@ -437,6 +437,33 @@ def test_total_usage_and_cost_normalize_mixed_responses_and_chat(): assert result.cost == pytest.approx((30 * 0.00125) + (12 * 0.005)) +def test_total_cost_applies_the_long_context_batch_tier_per_line(): + long_row = _success_row(usage=_usage(300_000, 10)) + short_row = _success_row(usage=_usage(100, 10)) + + result = bu._aggregate_batch_cost_usage_models( + entries=[long_row, short_row], + custom_llm_provider="openai", + model_info=ModelInfo( + key="lit-batch-tier", + max_tokens=None, + max_input_tokens=None, + max_output_tokens=None, + input_cost_per_token=2e-6, + output_cost_per_token=8e-6, + litellm_provider="openai", + mode="chat", + supported_openai_params=None, + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + input_cost_per_token_above_272k_tokens_batches=2e-6, + output_cost_per_token_above_272k_tokens_batches=6e-6, + ), + ) + + assert result.cost == pytest.approx((300_000 * 2e-6) + (10 * 6e-6) + (100 * 1e-6) + (10 * 4e-6)) + + def test_total_usage_empty_is_zero(): result = bu._aggregate_batch_cost_usage_models(entries=[], custom_llm_provider="openai") assert result.cost == 0.0 @@ -1824,6 +1851,45 @@ def test_unparsable_bedrock_batch_usage_warns(caplog): assert "inputTextTokenCount" in caplog.text +def test_total_cost_bills_cached_tokens_per_line_at_the_batch_cached_rate(): + responses_row = _success_row( + usage={ + "input_tokens": 300_000, + "output_tokens": 10, + "total_tokens": 300_010, + "input_tokens_details": {"cached_tokens": 299_000}, + } + ) + chat_row = _success_row(usage={**_usage(100, 10), "prompt_tokens_details": {"cached_tokens": 60}}) + + result = bu._aggregate_batch_cost_usage_models( + entries=[responses_row, chat_row], + custom_llm_provider="openai", + model_info=ModelInfo( + key="lit-batch-cached-tier", + max_tokens=None, + max_input_tokens=None, + max_output_tokens=None, + input_cost_per_token=2e-6, + output_cost_per_token=8e-6, + cache_read_input_token_cost=1e-6, + litellm_provider="openai", + mode="chat", + supported_openai_params=None, + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + cache_read_input_token_cost_batches=5e-7, + input_cost_per_token_above_272k_tokens_batches=2e-6, + output_cost_per_token_above_272k_tokens_batches=6e-6, + cache_read_input_token_cost_above_272k_tokens_batches=1e-6, + ), + ) + + long_line = 1_000 * 2e-6 + 299_000 * 1e-6 + 10 * 6e-6 + short_line = 40 * 1e-6 + 60 * 5e-7 + 10 * 4e-6 + assert result.cost == pytest.approx(long_line + short_line) + + # --------------------------------------------------------------------------- # # batch_cost_is_final # --------------------------------------------------------------------------- # diff --git a/tests/unit/chat_completions/test_dispatch.py b/tests/unit/chat_completions/test_dispatch.py index 63821c74208..2807ed7f8f7 100644 --- a/tests/unit/chat_completions/test_dispatch.py +++ b/tests/unit/chat_completions/test_dispatch.py @@ -10,7 +10,7 @@ from litellm.chat_completions.dispatch import ( ) from litellm.rust_bridge import catalog from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Route, Rule +from litellm.rust_bridge.catalog import Route, RouteRule from litellm.rust_bridge.chat_completions.entrypoints import ( NATIVE_ACOMPLETION, NATIVE_COMPLETION, @@ -23,7 +23,7 @@ from litellm.types.utils import ModelResponse MESSAGES: Final = [{"role": "user", "content": "hi"}] PYTHON_RULES: Final = () -RUST_RULES: Final = (Rule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED),) +RUST_RULES: Final = (RouteRule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED),) def completion_binding(native: NativeCompletion | None) -> NativeBinding[NativeCompletion]: @@ -117,9 +117,7 @@ def test_native_receives_bound_request_and_original_call_shape() -> None: "custom_llm_provider": "anthropic", "metadata": metadata, } - captured: Final[ - list[tuple[LiteLLMChatCompletionsRequest, tuple[object, ...], Mapping[str, object]]] - ] = [] + captured: Final[list[tuple[LiteLLMChatCompletionsRequest, tuple[object, ...], Mapping[str, object]]]] = [] def python(*call_args: object, **call_kwargs: object) -> ModelResponse: # kwargs-ok: rejected Rust fallback pytest.fail("Required Rust dispatch must not call Python") diff --git a/tests/unit/enterprise/enterprise_callbacks/test_secret_detection.py b/tests/unit/enterprise/enterprise_callbacks/test_secret_detection.py index bf9eeeb9968..405136924e7 100644 --- a/tests/unit/enterprise/enterprise_callbacks/test_secret_detection.py +++ b/tests/unit/enterprise/enterprise_callbacks/test_secret_detection.py @@ -10,16 +10,20 @@ Covers the three defects from the ticket: handling live only on the native path). """ +import cProfile +import sys import tempfile import time +from collections.abc import Callable +from typing import Final import pytest - from litellm_enterprise.enterprise_callbacks.secret_detection import ( - _ENTERPRISE_SecretDetection, _default_detect_secrets_config, + _ENTERPRISE_SecretDetection, _masked_entity_count, ) + from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth @@ -665,14 +669,47 @@ def test_scan_message_stays_linear_on_repeated_sk_separators(): assert time.perf_counter() - started < 2.0 +_SCALE: Final = 4 + + +def _value_run(n: int) -> str: + return f"api_key: '{OPENAI_KEY}'\npassword=" + "a-" * n + "!" + + +def _quote_run(n: int) -> str: + return f"api_key: '{OPENAI_KEY}'\npassword:" + '"' * n + + +def _keyword_run(n: int) -> str: + return f"api_key: '{OPENAI_KEY}'\n" + "api_key:" * n + + +def _value_repeat(n: int) -> str: + return f"api_key: '{OPENAI_KEY}'\nsecret=" + "aB3dE6gH9jK2mN5p " * n + + +def _assignment_flood(n: int) -> str: + return f"api_key: '{OPENAI_KEY}'\n" + "\n".join(f"password{i}=aB3dE6gH9jK2mN5p{i}" for i in range(n)) + + +def _calls_to_redact(guardrail: _ENTERPRISE_SecretDetection, content: str) -> int: + previous_profiler: Final = sys.getprofile() + profile: Final = cProfile.Profile() + try: + profile.runcall(guardrail.redact_text, content) + finally: + sys.setprofile(previous_profiler) + return sum(entry.callcount for entry in profile.getstats()) + + @pytest.mark.parametrize( - "content", + ("adversarial_content", "size"), [ - f"api_key: '{OPENAI_KEY}'\npassword=" + "a-" * 10_000 + "!", - f"api_key: '{OPENAI_KEY}'\npassword:" + '"' * 20_000, - f"api_key: '{OPENAI_KEY}'\n" + "api_key:" * 10_000, - f"api_key: '{OPENAI_KEY}'\nsecret=" + "aB3dE6gH9jK2mN5p " * 2_000, - f"api_key: '{OPENAI_KEY}'\n" + "\n".join(f"password{i}=aB3dE6gH9jK2mN5p{i}" for i in range(3_000)), + (_value_run, 10_000), + (_quote_run, 20_000), + (_keyword_run, 10_000), + (_value_repeat, 2_000), + (_assignment_flood, 3_000), ], ids=[ "value-run", @@ -682,12 +719,15 @@ def test_scan_message_stays_linear_on_repeated_sk_separators(): "assignment-flood", ], ) -def test_scan_message_stays_linear_on_adversarial_credential_lines(content): - guardrail = _guardrail() +def test_scan_message_stays_linear_on_adversarial_credential_lines( + adversarial_content: Callable[[int], str], size: int +) -> None: + guardrail: Final = _guardrail() - started = time.perf_counter() - guardrail.redact_text(content) - assert time.perf_counter() - started < 10.0 + small: Final = _calls_to_redact(guardrail, adversarial_content(size // _SCALE)) + large: Final = _calls_to_redact(guardrail, adversarial_content(size)) + + assert large <= 1.25 * _SCALE * small, (small, large) def test_scan_message_redacts_whole_stripe_live_key(): diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py index 50409b2ea2c..6f297e6e06a 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py @@ -7,10 +7,14 @@ through _hidden_params to the x-litellm-callback-duration-ms response header. import asyncio import datetime +import time +from collections.abc import Iterator +from typing import Final from unittest.mock import MagicMock import pytest +import litellm import litellm.litellm_core_utils.llm_response_utils.response_metadata as response_metadata_mod import litellm.proxy.common_request_processing as common_request_processing_mod from litellm.litellm_core_utils.litellm_logging import Logging @@ -22,7 +26,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.utils import ModelResponse +from litellm.types.utils import ModelResponse, Usage class TestCallbackDurationMs: @@ -425,6 +429,15 @@ class TestCallbackDurationInCustomHeaders: assert "x-litellm-callback-duration-ms" not in headers +@pytest.fixture(params=["UTC", "Asia/Kolkata", "America/Los_Angeles"]) +def process_timezone(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: + monkeypatch.setenv("TZ", request.param) + time.tzset() + yield request.param + monkeypatch.undo() + time.tzset() + + class TestDetailedTiming: """Tests for detailed per-phase timing headers behind LITELLM_DETAILED_TIMING.""" @@ -470,13 +483,14 @@ class TestDetailedTiming: assert hidden.get("timing_pre_processing_ms") == 20.0 assert hidden.get("timing_post_processing_ms") == 10.0 # 530 - 20 - 500 - def test_detailed_timing_pre_processing_uses_receive_anchor(self, monkeypatch): + @pytest.mark.skipif(not hasattr(time, "tzset"), reason="switching the process timezone needs time.tzset()") + def test_detailed_timing_pre_processing_uses_receive_anchor(self, monkeypatch, process_timezone: str): monkeypatch.setattr(response_metadata_mod, "LITELLM_DETAILED_TIMING", True) result = ModelResponse() received_at = datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc) start = received_at + datetime.timedelta(milliseconds=200) - api_call_start = start.replace(tzinfo=None) + api_call_start = start.astimezone().replace(tzinfo=None) end = start + datetime.timedelta(milliseconds=530) logging_obj = self._make_logging_obj( llm_api_duration_ms=500.0, @@ -583,3 +597,57 @@ class TestLoggingInitCallbackDuration: # Should still be set (deep copy of None is essentially a no-op) assert hasattr(obj, "callback_duration_ms") assert obj.callback_duration_ms >= 0 + + +def test_update_response_metadata_prices_per_second_deployment_from_its_stamped_duration(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + deployment_id: Final = "per-second-deployment-response-metadata" + litellm.register_model( + model_cost={ + deployment_id: { + "input_cost_per_second": 0.02, + "output_cost_per_second": 0.04, + "litellm_provider": "openai", + "mode": "chat", + } + } + ) + start_time: Final = datetime.datetime(2026, 9, 21, 12, 0, 0) + logging_obj: Final = Logging( + model="gpt-5.4-nano", + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=start_time, + litellm_call_id="per-second-response-metadata", + function_id="f", + ) + logging_obj.update_environment_variables( + model="gpt-5.4-nano", + litellm_params={ + "input_cost_per_second": 0.02, + "output_cost_per_second": 0.04, + "metadata": {"model_info": {"id": deployment_id}}, + }, + optional_params={}, + custom_llm_provider="openai", + ) + logging_obj.model_call_details["end_time"] = start_time + datetime.timedelta(seconds=10) + result: Final = ModelResponse( + model="gpt-5.4-nano", + usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18), + ) + + update_response_metadata( + result=result, + logging_obj=logging_obj, + model="gpt-5.4-nano", + kwargs={"model_info": {"id": deployment_id}}, + start_time=start_time, + end_time=start_time + datetime.timedelta(seconds=2), + ) + + assert result._response_ms == pytest.approx(2000) + assert result._hidden_params["response_cost"] == pytest.approx((0.02 + 0.04) * 2) diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index e4a33d5772c..47609261a25 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -157,27 +157,6 @@ def test_foundry_gpt_6_astra_keeps_sampling_params_when_reasoning_effort_is_none assert optional_params == {"reasoning_effort": "none", "temperature": 0.2, "top_p": 0.9} -def test_a_gpt_5_name_without_a_foundry_row_keeps_reading_its_own_entry( - monkeypatch: pytest.MonkeyPatch, _local_model_cost_map -): - """Most gpt-5-family names have no azure_ai/ row. Reading an azure_ai/ key for those finds - nothing, and an openai.azure.com base sends the name down the azure provider, which has no key - for it either, so every effort answer would silently fall back to false and take temperature, - top_p and logprobs down with it.""" - monkeypatch.setenv("AZURE_AI_API_BASE", "https://example-resource.openai.azure.com") - monkeypatch.setenv("AZURE_AI_API_KEY", "placeholder") - - optional_params = litellm.utils.get_optional_params( - model="gpt-5.1-chat-latest", - custom_llm_provider="azure_ai", - temperature=0.2, - top_p=0.9, - logprobs=True, - ) - - assert optional_params["temperature"] == 0.2 - assert optional_params["top_p"] == 0.9 - assert optional_params["logprobs"] is True def test_azure_ai_grok_stop_parameter_handling(): diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_handler.py b/tests/unit/llms/bedrock/files/test_bedrock_files_handler.py index 639be272351..5c078affffc 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_handler.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_handler.py @@ -270,3 +270,40 @@ async def test_afile_content_assumes_role_with_external_id(monkeypatch): assert s3_client_kwargs["aws_access_key_id"] == "ASIAFILESDOWNLOADROLE" assert s3_client_kwargs["aws_session_token"] == "assumed-session-token" assert response.content == b'{"custom_id": "req-1"}' + + +@pytest.mark.asyncio +async def test_afile_content_builds_the_s3_client_with_the_s3_pair_when_it_differs_from_the_aws_identity(): + import boto3 + + class FakeS3Body: + def read(self): + return b'{"custom_id": "req-1"}' + + class FakeS3Client: + def get_object(self, Bucket, Key): + return {"Body": FakeS3Body()} + + optional_params = { + "_litellm_internal_model_credentials": MappingProxyType({"s3_bucket_name": "safe-bucket"}), + "aws_region_name": "us-east-1", + "aws_access_key_id": "AKIABEDROCKONLY", + "aws_secret_access_key": "bedrock-only-secret", + "aws_session_token": "bedrock-only-token", + "s3_access_key_id": "AKIAS3ONLY", + "s3_secret_access_key": "s3-only-secret", + } + + with patch.object(boto3, "client", return_value=FakeS3Client()) as mock_boto3_client: + response = await BedrockFilesHandler().afile_content( + file_content_request={"file_id": "s3://safe-bucket/litellm-bedrock-files-model-id-abc.jsonl"}, + optional_params=optional_params, + timeout=10.0, + max_retries=None, + ) + + s3_client_kwargs = mock_boto3_client.call_args.kwargs + assert s3_client_kwargs["aws_access_key_id"] == "AKIAS3ONLY" + assert s3_client_kwargs["aws_secret_access_key"] == "s3-only-secret" + assert s3_client_kwargs["aws_session_token"] is None, "the aws_* session token belongs to the Bedrock identity" + assert response.content == b'{"custom_id": "req-1"}' diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 2d2de77269b..d0921e68424 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -6,6 +6,7 @@ import json import os from collections.abc import Mapping from contextlib import AsyncExitStack, closing +from types import MappingProxyType from typing import Final from unittest.mock import MagicMock from urllib.parse import unquote, urlparse @@ -3789,3 +3790,82 @@ class TestBedrockFileListTransformation: assert denied.value.status_code == 403 assert "AccessDenied" in denied.value.message + + +_SPLIT_IDENTITY_PARAMS: Final = { + "aws_region_name": "us-east-1", + "aws_access_key_id": "AKIABEDROCKONLY", + "aws_secret_access_key": "bedrock-only-secret", + "s3_access_key_id": "AKIAS3ONLY", + "s3_secret_access_key": "s3-only-secret", + "s3_bucket_name": "safe-bucket", +} + + +def _authorization(headers: Mapping[str, str]) -> str: + return {key.lower(): value for key, value in headers.items()}["authorization"] + + +def test_sign_s3_request_uses_the_s3_pair_when_it_differs_from_the_aws_identity(): + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + signed_headers, _signed_body = BedrockFilesConfig()._sign_s3_request( + content='{"custom_id": "req-1"}', + api_base="https://s3.us-east-1.amazonaws.com/safe-bucket/litellm-bedrock-files-model-id-abc.jsonl", + optional_params=dict(_SPLIT_IDENTITY_PARAMS), + ) + + assert _authorization(signed_headers).startswith("AWS4-HMAC-SHA256 Credential=AKIAS3ONLY/"), ( + "the S3 PutObject must be signed by s3_access_key_id, not the Bedrock aws_access_key_id" + ) + + +def test_sign_s3_request_with_the_s3_pair_ignores_ambient_aws_session_token_role_and_profile(monkeypatch): + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + monkeypatch.setenv("AWS_SESSION_TOKEN", "pod-token") + monkeypatch.setenv("AWS_ROLE_NAME", "arn:aws:iam::123456789012:role/pod") + monkeypatch.setenv("AWS_PROFILE_NAME", "pod-profile") + signed_headers, _signed_body = BedrockFilesConfig()._sign_s3_request( + content='{"custom_id": "req-1"}', + api_base="https://s3.us-east-1.amazonaws.com/safe-bucket/litellm-bedrock-files-model-id-abc.jsonl", + optional_params=dict(_SPLIT_IDENTITY_PARAMS), + ) + + lowered: Final = {key.lower(): value for key, value in signed_headers.items()} + assert lowered["authorization"].startswith("AWS4-HMAC-SHA256 Credential=AKIAS3ONLY/") + assert "x-amz-security-token" not in lowered, "an ambient AWS_SESSION_TOKEN must not be mixed into the s3_* pair" + + +@pytest.mark.parametrize("method", ["GET", "DELETE"]) +def test_sign_s3_request_without_body_uses_the_s3_pair_when_it_differs_from_the_aws_identity(method): + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig, _BedrockS3RequestParams + + signed_headers = BedrockFilesConfig()._sign_s3_request_without_body( + method=method, + api_base="https://s3.us-east-1.amazonaws.com/safe-bucket/litellm-bedrock-files-model-id-abc.jsonl", + aws_region_name="us-east-1", + request_params=_BedrockS3RequestParams.model_validate(_SPLIT_IDENTITY_PARAMS), + ) + + assert _authorization(signed_headers).startswith("AWS4-HMAC-SHA256 Credential=AKIAS3ONLY/"), ( + f"the S3 {method} must be signed by s3_access_key_id, not the Bedrock aws_access_key_id" + ) + + +def test_transform_file_content_request_signs_with_the_s3_pair_from_litellm_params(): + from litellm.llms.bedrock.files.transformation import S3_SIGNED_REQUEST_HEADERS_PARAM, BedrockFilesConfig + + litellm_params = { + **_SPLIT_IDENTITY_PARAMS, + "_litellm_internal_model_credentials": MappingProxyType({"s3_bucket_name": "safe-bucket"}), + } + BedrockFilesConfig().transform_file_content_request( + file_content_request={"file_id": "s3://safe-bucket/litellm-bedrock-files-model-id-abc.jsonl"}, + optional_params={}, + litellm_params=litellm_params, + ) + + assert _authorization(litellm_params[S3_SIGNED_REQUEST_HEADERS_PARAM]).startswith( + "AWS4-HMAC-SHA256 Credential=AKIAS3ONLY/" + ) diff --git a/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index d1d636a15f7..fbcbda183bc 100644 --- a/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -1,9 +1,13 @@ import base64 +import gc import json import struct -import tracemalloc +import sys +import types from binascii import crc32 +from collections.abc import Callable from datetime import datetime +from typing import Final from unittest.mock import patch from litellm.litellm_core_utils.litellm_logging import Logging @@ -50,10 +54,7 @@ def test_bedrock_passthrough_get_complete_url_default_endpoint(): ) # Verify URL construction - assert ( - str(url) - == "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-sonnet/invoke" - ) + assert str(url) == "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-sonnet/invoke" assert api_base == "https://bedrock-runtime.us-east-1.amazonaws.com" @@ -112,9 +113,7 @@ def test_bedrock_passthrough_get_complete_url_custom_endpoint_with_path(): model="anthropic.claude-3-sonnet", endpoint="/model/anthropic.claude-3-sonnet/invoke", request_query_params=None, - litellm_params={ - "aws_bedrock_runtime_endpoint": "http://proxy.com/bedrockproxy" - }, + litellm_params={"aws_bedrock_runtime_endpoint": "http://proxy.com/bedrockproxy"}, ) # Verify get_runtime_endpoint was called with correct parameters @@ -126,10 +125,7 @@ def test_bedrock_passthrough_get_complete_url_custom_endpoint_with_path(): ) # Verify URL construction preserves the proxy path - assert ( - str(url) - == "http://proxy.com/bedrockproxy/model/anthropic.claude-3-sonnet/invoke" - ) + assert str(url) == "http://proxy.com/bedrockproxy/model/anthropic.claude-3-sonnet/invoke" assert api_base == "http://proxy.com/bedrockproxy" @@ -211,9 +207,7 @@ def test_bedrock_passthrough_with_application_inference_profile(): config = BedrockPassthroughConfig() model = "anthropic.claude-sonnet-4-20250514-v1:0" - model_id = ( - "arn:aws:bedrock:eu-west-1:123456789:application-inference-profile/abcdefgh1234" - ) + model_id = "arn:aws:bedrock:eu-west-1:123456789:application-inference-profile/abcdefgh1234" endpoint = f"model/{model}/invoke" with ( @@ -239,12 +233,8 @@ def test_bedrock_passthrough_with_application_inference_profile(): # Verify that the URL contains the encoded model_id (ARN) instead of the model name url_str = str(url) # The ARN slash should be encoded as %2F - assert ( - "application-inference-profile%2F" in url_str - ), f"Expected encoded ARN in URL, but got: {url_str}" - assert ( - model not in url_str - ), f"Model name should be replaced by model_id, but got: {url_str}" + assert "application-inference-profile%2F" in url_str, f"Expected encoded ARN in URL, but got: {url_str}" + assert model not in url_str, f"Model name should be replaced by model_id, but got: {url_str}" assert "/invoke" in url_str, "Expected /invoke action in URL" # Verify the complete URL structure with encoded ARN @@ -258,9 +248,7 @@ def test_bedrock_passthrough_with_inference_profile_converse_endpoint(): config = BedrockPassthroughConfig() model = "anthropic.claude-sonnet-4-20250514-v1:0" - model_id = ( - "arn:aws:bedrock:us-east-1:123456789:application-inference-profile/xyz123" - ) + model_id = "arn:aws:bedrock:us-east-1:123456789:application-inference-profile/xyz123" endpoint = f"model/{model}/converse" with ( @@ -323,12 +311,8 @@ def test_bedrock_passthrough_without_model_id_backward_compatibility(): # Verify that the URL contains the model name (not replaced) url_str = str(url) - assert ( - model in url_str - ), f"Expected model name in URL when model_id not provided, but got: {url_str}" - expected_url = ( - f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model}/invoke" - ) + assert model in url_str, f"Expected model name in URL when model_id not provided, but got: {url_str}" + expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model}/invoke" assert url_str == expected_url @@ -338,9 +322,7 @@ def test_bedrock_passthrough_region_extraction_from_inference_profile_arn(): model = "anthropic.claude-sonnet-4-20250514-v1:0" # ARN contains us-west-2 region - model_id = ( - "arn:aws:bedrock:us-west-2:123456789:application-inference-profile/test123" - ) + model_id = "arn:aws:bedrock:us-west-2:123456789:application-inference-profile/test123" endpoint = f"model/{model}/invoke" # Don't provide aws_region_name in litellm_params to test ARN extraction @@ -358,15 +340,13 @@ def test_bedrock_passthrough_region_extraction_from_inference_profile_arn(): model=model, endpoint=endpoint, request_query_params=None, - litellm_params={ - "model_id": model_id - }, # Region should be extracted from ARN + litellm_params={"model_id": model_id}, # Region should be extracted from ARN ) # Verify that the region from ARN is used in the base URL - assert ( - "us-west-2" in api_base - ), f"Expected region 'us-west-2' from ARN in base URL, but got: {api_base}" + assert "us-west-2" in api_base, f"Expected region 'us-west-2' from ARN in base URL, but got: {api_base}" + + def test_bedrock_passthrough_model_id_arn_encoding(): """ Test that model_id ARNs are properly URL-encoded when used in endpoints. @@ -409,14 +389,14 @@ def test_bedrock_passthrough_model_id_arn_encoding(): url_str = str(url) # The slash in the ARN after application-inference-profile should be encoded as %2F - assert ( - "application-inference-profile%2F" in url_str - ), f"Expected encoded ARN with %2F in URL, but got: {url_str}" + assert "application-inference-profile%2F" in url_str, ( + f"Expected encoded ARN with %2F in URL, but got: {url_str}" + ) # The unencoded version should NOT be in the URL - assert ( - "application-inference-profile/" not in url_str - ), f"ARN slash should be encoded, but found unencoded version in: {url_str}" + assert "application-inference-profile/" not in url_str, ( + f"ARN slash should be encoded, but found unencoded version in: {url_str}" + ) # Verify the complete expected URL structure expected_encoded_model_id = ( @@ -433,9 +413,7 @@ def test_bedrock_passthrough_model_id_arn_encoding_invoke_endpoint(): config = BedrockPassthroughConfig() model = "anthropic.claude-sonnet-4-5-20250929-v1:0" - model_id = ( - "arn:aws:bedrock:us-east-1:123456789:application-inference-profile/xyz789" - ) + model_id = "arn:aws:bedrock:us-east-1:123456789:application-inference-profile/xyz789" endpoint = f"/model/{model}/invoke" with ( @@ -464,9 +442,7 @@ def test_bedrock_passthrough_model_id_arn_encoding_invoke_endpoint(): assert "application-inference-profile%2F" in url_str assert "/invoke" in url_str - expected_encoded_model_id = ( - "arn:aws:bedrock:us-east-1:123456789:application-inference-profile%2Fxyz789" - ) + expected_encoded_model_id = "arn:aws:bedrock:us-east-1:123456789:application-inference-profile%2Fxyz789" expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{expected_encoded_model_id}/invoke" assert url_str == expected_url @@ -508,9 +484,7 @@ def test_bedrock_passthrough_model_id_without_arn(): assert model_id in url_str assert "%2F" not in url_str, "Non-ARN model IDs should not be encoded" - expected_url = ( - f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model_id}/converse" - ) + expected_url = f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model_id}/converse" assert url_str == expected_url @@ -518,8 +492,7 @@ def _event_frame(event_type: str, payload: dict) -> bytes: def header(name: str, value: str) -> bytes: name_b, value_b = name.encode(), value.encode() return ( - struct.pack("!B", len(name_b)) + name_b - + struct.pack("!B", 7) + struct.pack("!H", len(value_b)) + value_b + struct.pack("!B", len(name_b)) + name_b + struct.pack("!B", 7) + struct.pack("!H", len(value_b)) + value_b ) payload_b = json.dumps(payload, separators=(",", ":")).encode() @@ -592,29 +565,57 @@ def _feed(collector: PassthroughStreamCollector, stream: bytes, chunk_size: int collector.add(stream[offset : offset + chunk_size]) -def test_converse_stream_collector_keeps_usage_without_retaining_the_stream(): - texts = [f"tok{i} " for i in range(4000)] - stream = ( - _event_frame("messageStart", {"role": "assistant"}) + _text_block(0, texts) + _stream_tail("end_turn", 4000) +_SHARED_OBJECT_TYPES: Final = (types.ModuleType, type, types.FunctionType, types.MethodType, types.CodeType, property) +_SMALL_TOKENS: Final = 4000 +_LARGE_TOKENS: Final = 8000 + + +def _bytes_reachable_from(root: object) -> int: + seen: Final[set[int]] = set() + pending: Final[list[object]] = [root] + sizes: Final[list[int]] = [] + while pending: + obj = pending.pop() + if id(obj) in seen or isinstance(obj, _SHARED_OBJECT_TYPES): + continue + seen.add(id(obj)) + sizes.append(sys.getsizeof(obj)) + pending.extend(gc.get_referents(obj)) + return sum(sizes) + + +def _converse_text_stream(texts: list[str]) -> bytes: + return ( + _event_frame("messageStart", {"role": "assistant"}) + + _text_block(0, texts) + + _stream_tail("end_turn", len(texts)) ) - _feed(_converse_stream_collector(), stream) - tracemalloc.start() - try: - base = tracemalloc.get_traced_memory()[0] - collector = _converse_stream_collector() - _feed(collector, stream) - retained = tracemalloc.get_traced_memory()[0] - base - finally: - tracemalloc.stop() - assert retained < len(stream) // 4 +def _retained_growth(build_stream: Callable[[list[str]], bytes], endpoint: str) -> tuple[int, int, list[str]]: + small_texts: Final = [f"tok{i} " for i in range(_SMALL_TOKENS)] + large_texts: Final = [f"tok{i} " for i in range(_LARGE_TOKENS)] + small_stream, large_stream = build_stream(small_texts), build_stream(large_texts) + small, large = _stream_collector(endpoint), _stream_collector(endpoint) + _feed(small, small_stream) + _feed(large, large_stream) + growth: Final = _bytes_reachable_from(large) - _bytes_reachable_from(small) + return growth, len(large_stream) - len(small_stream), large_texts + +def test_converse_stream_collector_keeps_usage_without_retaining_the_stream(): + growth, extra_stream_bytes, texts = _retained_growth(_converse_text_stream, CONVERSE_STREAM_ENDPOINT) + extra_text_bytes = sum(len(text) for text in texts[_SMALL_TOKENS:]) + + assert extra_text_bytes <= growth < extra_stream_bytes // 4, (growth, extra_text_bytes, extra_stream_bytes) + + collector = _converse_stream_collector() + _feed(collector, _converse_text_stream(texts)) response = collector.build_logged_response(_converse_stream_logging_obj()) assert isinstance(response, ModelResponse) assert response.choices[0].message.content == "".join(texts) assert response.choices[0].finish_reason == "stop" - assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (25, 4000) + assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (25, _LARGE_TOKENS) def test_converse_stream_collector_keeps_tool_calls_between_text_runs(): @@ -645,9 +646,8 @@ def test_converse_stream_collector_keeps_tool_calls_between_text_runs(): assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (25, 12) -def test_invoke_stream_collector_keeps_usage_without_retaining_the_stream(): - texts = [f"tok{i} " for i in range(4000)] - stream = ( +def _invoke_text_stream(texts: list[str]) -> bytes: + return ( _invoke_chunk( { "type": "message_start", @@ -669,28 +669,25 @@ def test_invoke_stream_collector_keeps_usage_without_retaining_the_stream(): ) + _invoke_chunk({"type": "content_block_stop", "index": 0}) + _invoke_chunk( - {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 4000}} + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": len(texts)}} ) + _invoke_chunk({"type": "message_stop"}) ) - _feed(_stream_collector(INVOKE_STREAM_ENDPOINT), stream) - tracemalloc.start() - try: - base = tracemalloc.get_traced_memory()[0] - collector = _stream_collector(INVOKE_STREAM_ENDPOINT) - _feed(collector, stream) - retained = tracemalloc.get_traced_memory()[0] - base - finally: - tracemalloc.stop() - assert retained < len(stream) // 4 +def test_invoke_stream_collector_keeps_usage_without_retaining_the_stream(): + growth, extra_stream_bytes, texts = _retained_growth(_invoke_text_stream, INVOKE_STREAM_ENDPOINT) + extra_text_bytes = sum(len(text) for text in texts[_SMALL_TOKENS:]) + assert extra_text_bytes <= growth < extra_stream_bytes // 4, (growth, extra_text_bytes, extra_stream_bytes) + + collector = _stream_collector(INVOKE_STREAM_ENDPOINT) + _feed(collector, _invoke_text_stream(texts)) response = collector.build_logged_response(_stream_logging_obj(INVOKE_STREAM_ENDPOINT)) assert isinstance(response, ModelResponse) assert response.choices[0].message.content == "".join(texts) assert response.choices[0].finish_reason == "stop" - assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (25, 4000) + assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (25, _LARGE_TOKENS) def test_stream_collector_logs_nothing_for_an_unrecognized_endpoint(): diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 6815f00267c..3f740edf834 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1695,21 +1695,6 @@ def test_nim_vllm_extras_translated_end_to_end_in_request_body(): assert request_body["top_k"] == 40 -def test_in_schema_unsupported_params_still_raise(): - with pytest.raises(litellm.UnsupportedParamsError): - litellm.get_optional_params( - model="accounts/fireworks/models/llama-v3-70b-instruct", - custom_llm_provider="fireworks_ai", - drop_params=False, - store=True, - ) - optional_params = litellm.get_optional_params( - model="accounts/fireworks/models/llama-v3-70b-instruct", - custom_llm_provider="fireworks_ai", - drop_params=True, - store=True, - ) - assert "store" not in optional_params def test_streaming_preserves_selected_model_for_private_accounting(): diff --git a/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py b/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py index c39affc18a8..4e7a2931cda 100644 --- a/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py +++ b/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py @@ -715,7 +715,7 @@ class TestMoonshotReasoningEffort: def force_local_model_cost(self, monkeypatch): monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map()) - @pytest.mark.parametrize("model", ["kimi-k3", "kimi-k2.5", "kimi-k2.6", "kimi-k2-thinking"]) + @pytest.mark.parametrize("model", ["kimi-k3", "kimi-k2.5", "kimi-k2.6"]) def test_reasoning_model_supports_reasoning_effort(self, model): assert "reasoning_effort" in MoonshotChatConfig().get_supported_openai_params(model) diff --git a/tests/unit/messages/test_dispatch.py b/tests/unit/messages/test_dispatch.py index 586b77d9a25..48eb1adbf51 100644 --- a/tests/unit/messages/test_dispatch.py +++ b/tests/unit/messages/test_dispatch.py @@ -12,7 +12,7 @@ from litellm.messages.dispatch import ( ) from litellm.rust_bridge import catalog from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Route, Rule, Rules +from litellm.rust_bridge.catalog import Route, RouteRule, Rules from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.messages.entrypoints import ( NATIVE_AMESSAGES, @@ -25,7 +25,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMe MESSAGES: Final = [{"role": "user", "content": "hi"}] PYTHON_RULES: Final[Rules] = () -RUST_RULES: Final[Rules] = (Rule(Route.MESSAGES, Rollout.RUST_REQUIRED),) +RUST_RULES: Final[Rules] = (RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED),) def messages_binding(native: NativeMessages | None) -> NativeBinding[NativeMessages]: diff --git a/tests/unit/ocr/test_dispatch.py b/tests/unit/ocr/test_dispatch.py index e54d4070ba8..2727ff23449 100644 --- a/tests/unit/ocr/test_dispatch.py +++ b/tests/unit/ocr/test_dispatch.py @@ -12,7 +12,7 @@ from litellm.ocr.dispatch import ( ) from litellm.rust_bridge import catalog from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Route, Rule, Rules +from litellm.rust_bridge.catalog import Route, RouteRule, Rules from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.ocr.entrypoints import ( NATIVE_AOCR, @@ -22,8 +22,8 @@ from litellm.rust_bridge.ocr.entrypoints import ( NativeOcr, ) -PYTHON_RULES: Final[Rules] = (Rule(Route.OCR, Rollout.PYTHON_ONLY),) -RUST_RULES: Final[Rules] = (Rule(Route.OCR, Rollout.RUST_REQUIRED),) +PYTHON_RULES: Final[Rules] = (RouteRule(Route.OCR, Rollout.PYTHON_ONLY),) +RUST_RULES: Final[Rules] = (RouteRule(Route.OCR, Rollout.RUST_REQUIRED),) def ocr_binding(native: NativeOcr | None) -> NativeBinding[NativeOcr]: @@ -403,8 +403,8 @@ def test_provider_scoped_rule_sees_the_provider_named_by_the_model_prefix( model: str, custom_llm_provider: str | None, expected: str ) -> None: rules: Final[Rules] = ( - Rule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})), - Rule(Route.OCR, Rollout.PYTHON_ONLY), + RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})), + RouteRule(Route.OCR, Rollout.PYTHON_ONLY), ) document: Final[Mapping[str, object]] = {"type": "image_url", "image_url": "data:image/png;base64,YQ=="} kwargs: Final[Mapping[str, object]] = ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyInfo.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyInfo.test.ts new file mode 100644 index 00000000000..96e9cff6ac7 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyInfo.test.ts @@ -0,0 +1,101 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { renderHook, waitFor } from "@testing-library/react"; +import React, { ReactNode } from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { keyInfoV1Call, userGetInfoV2 } from "@/components/networking"; + +import { useKeyInfo } from "./useKeyInfo"; + +vi.mock("@/components/networking", () => ({ + keyInfoV1Call: vi.fn(), + userGetInfoV2: vi.fn(), +})); + +const mockUseAuthorized = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => mockUseAuthorized(), +})); + +const mockKeyInfoV1Call = vi.mocked(keyInfoV1Call); +const mockUserGetInfoV2 = vi.mocked(userGetInfoV2); + +const createWrapper = () => { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + const wrapper = ({ children }: { children: ReactNode }) => + React.createElement(QueryClientProvider, { client: queryClient }, children); + return { queryClient, wrapper }; +}; + +const KEY_ID = "sk-key-1"; +const ACCESS_TOKEN = "sk-access"; + +const OWNER = { + user_id: "user-1", + user_email: "owner@example.com", + user_alias: "Budget Owner", + user_role: "user", + spend: 0, + max_budget: 1500, + models: [], + budget_duration: "1mo", + budget_reset_at: null, + metadata: null, + created_at: null, + updated_at: null, + sso_user_id: null, + teams: [], +}; + +const keyInfoResponse = (info: Record) => ({ info }); + +describe("useKeyInfo", () => { + beforeEach(() => { + mockKeyInfoV1Call.mockReset(); + mockUserGetInfoV2.mockReset(); + mockUseAuthorized.mockReturnValue({ accessToken: ACCESS_TOKEN }); + }); + + it("attaches the owner's budget fields when the key has a user_id", async () => { + mockKeyInfoV1Call.mockResolvedValue(keyInfoResponse({ user_id: "user-1", key_alias: "team-key" })); + mockUserGetInfoV2.mockResolvedValue(OWNER); + const { wrapper } = createWrapper(); + + const { result } = renderHook(() => useKeyInfo(KEY_ID), { wrapper }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(mockUserGetInfoV2).toHaveBeenCalledWith(ACCESS_TOKEN, "user-1"); + const expectedUser = { + user_id: "user-1", + user_email: "owner@example.com", + user_alias: "Budget Owner", + max_budget: 1500, + budget_duration: "1mo", + }; + expect(result.current.data?.user).toEqual(expectedUser); + }); + + it("does not fetch an owner when the key has no user_id", async () => { + mockKeyInfoV1Call.mockResolvedValue(keyInfoResponse({ user_id: null, key_alias: "service-key" })); + const { wrapper } = createWrapper(); + + const { result } = renderHook(() => useKeyInfo(KEY_ID), { wrapper }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(mockUserGetInfoV2).not.toHaveBeenCalled(); + expect(result.current.data?.user).toBeUndefined(); + }); + + it("still resolves the key data when the owner lookup fails", async () => { + mockKeyInfoV1Call.mockResolvedValue(keyInfoResponse({ user_id: "user-1" })); + mockUserGetInfoV2.mockRejectedValue(new Error("403")); + const { wrapper } = createWrapper(); + + const { result } = renderHook(() => useKeyInfo(KEY_ID), { wrapper }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(result.current.data?.token).toBe(KEY_ID); + expect(result.current.data?.api_key).toBe(KEY_ID); + expect(result.current.data?.user).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyInfo.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyInfo.ts index 57950953d74..be248e9aafb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyInfo.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyInfo.ts @@ -2,10 +2,25 @@ import { useQuery, UseQueryResult } from "@tanstack/react-query"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { KeyResponse } from "@/components/key_team_helpers/key_list"; -import { keyInfoV1Call } from "@/components/networking"; +import { keyInfoV1Call, userGetInfoV2 } from "@/components/networking"; import { keyKeys } from "./useKeys"; +const fetchOwner = async (accessToken: string, userId: string): Promise => { + try { + const owner = await userGetInfoV2(accessToken, userId); + return { + user_id: owner.user_id, + user_email: owner.user_email, + user_alias: owner.user_alias, + max_budget: owner.max_budget, + budget_duration: owner.budget_duration, + }; + } catch { + return undefined; + } +}; + export function useKeyInfo(keyId: string | null, options?: { enabled?: boolean }): UseQueryResult { const { accessToken } = useAuthorized(); @@ -14,10 +29,16 @@ export function useKeyInfo(keyId: string | null, options?: { enabled?: boolean } queryFn: async () => { if (!accessToken || !keyId) throw new Error("Missing access token or key id"); const keyData = await keyInfoV1Call(accessToken, keyId); + const info = keyData["info"]; + const owner = + typeof info.user_id === "string" && info.user_id !== "" + ? await fetchOwner(accessToken, info.user_id) + : undefined; return { - ...keyData["info"], + ...info, token: keyId, api_key: keyId, + ...(owner ? { user: owner } : {}), }; }, enabled: Boolean(accessToken && keyId) && (options?.enabled ?? true), diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.ts index b5adb0994e6..bb37b559cb6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/router/useRouterFields.ts @@ -16,6 +16,7 @@ export interface RouterSettingsField { export interface RouterFieldsResponse { fields: RouterSettingsField[]; routing_strategy_descriptions: Record; + routing_group_strategies?: string[]; } const routerFieldsKeys = createQueryKeys("routerFields"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/routingGroups/useRoutingGroups.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/routingGroups/useRoutingGroups.test.ts new file mode 100644 index 00000000000..e5f7814c7c8 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/routingGroups/useRoutingGroups.test.ts @@ -0,0 +1,65 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, renderHook, waitFor } from "@testing-library/react"; +import { createElement, type ReactNode } from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { getRouterSettingsCall, setCallbacksCall } from "@/components/networking"; +import type { RoutingGroup } from "@/components/routing_groups/types"; +import { useRoutingGroups, useSaveRoutingGroups } from "./useRoutingGroups"; + +vi.mock("@/components/networking", () => ({ + getRouterSettingsCall: vi.fn(), + setCallbacksCall: vi.fn(), +})); + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => ({ accessToken: "test-token", userId: "admin", userRole: "Admin" }), +})); + +const createWrapper = () => { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return function QueryWrapper({ children }: { children: ReactNode }) { + return createElement(QueryClientProvider, { client }, children); + }; +}; + +describe("routing group settings", () => { + beforeEach(() => vi.clearAllMocks()); + + it.each([ + { metadata: ["simple-shuffle", "priority"], expected: ["simple-shuffle", "priority"] }, + { metadata: undefined, expected: ["simple-shuffle"] }, + ])("uses the advertised group strategies and supports older gateways", async ({ metadata, expected }) => { + vi.mocked(getRouterSettingsCall).mockResolvedValue({ + fields: [{ field_name: "routing_strategy", options: ["simple-shuffle"] }], + routing_group_strategies: metadata, + current_values: {}, + }); + const { result } = renderHook(() => useRoutingGroups(), { wrapper: createWrapper() }); + + await waitFor(() => expect(result.current.data?.availableStrategies).toEqual(expected)); + }); + + it("loads and saves membership priorities through the existing router settings endpoint", async () => { + const groups: RoutingGroup[] = [ + { + group_name: "preferred-chat", + models: ["preferred", "backup"], + routing_strategy: "priority", + model_priorities: { preferred: 1, backup: 4 }, + }, + ]; + vi.mocked(getRouterSettingsCall).mockResolvedValue({ current_values: { routing_groups: groups } }); + vi.mocked(setCallbacksCall).mockResolvedValue({}); + const { result } = renderHook(() => ({ query: useRoutingGroups(), save: useSaveRoutingGroups() }), { + wrapper: createWrapper(), + }); + + await waitFor(() => expect(result.current.query.data?.routingGroups).toEqual(groups)); + await act(async () => { + await result.current.save.mutateAsync(groups); + }); + + expect(setCallbacksCall).toHaveBeenCalledWith("test-token", { router_settings: { routing_groups: groups } }); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/routingGroups/useRoutingGroups.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/routingGroups/useRoutingGroups.ts index 71d27d2017c..5ed6fa6a594 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/routingGroups/useRoutingGroups.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/routingGroups/useRoutingGroups.ts @@ -19,11 +19,12 @@ const fetchRoutingGroups = async (accessToken: string): Promise f?.field_name === "routing_strategy"); + const groupStrategies: unknown = data?.routing_group_strategies ?? routingStrategyField?.options; return { routingGroups: Array.isArray(currentValues.routing_groups) ? currentValues.routing_groups : [], routingStrategy: currentValues.routing_strategy ?? null, - availableStrategies: Array.isArray(routingStrategyField?.options) ? routingStrategyField.options : [], + availableStrategies: Array.isArray(groupStrategies) ? groupStrategies : [], }; }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/uiSettings/useApplyUserBudgetToTeamKeys.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/uiSettings/useApplyUserBudgetToTeamKeys.ts new file mode 100644 index 00000000000..3aeed5e4338 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/uiSettings/useApplyUserBudgetToTeamKeys.ts @@ -0,0 +1,6 @@ +import { useUISettings } from "./useUISettings"; + +export const APPLY_USER_BUDGET_TO_TEAM_KEYS_SETTING_KEY = "apply_user_budget_to_team_keys"; + +export const useApplyUserBudgetToTeamKeys = (): boolean => + useUISettings().data?.values?.[APPLY_USER_BUDGET_TO_TEAM_KEYS_SETTING_KEY] === true; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.test.tsx index da564f23de5..58e1e9bcce9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.test.tsx @@ -4,6 +4,8 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { MCPServerView } from "./mcp_server_view"; import * as networking from "@/components/networking"; +import { setSecureItem } from "@/utils/secureStorage"; +import { EDIT_OAUTH_UI_STATE_KEY } from "./mcp_server_edit"; import type { MCPServer } from "@/components/mcp_tools/types"; vi.mock(".", () => ({ @@ -68,6 +70,7 @@ const openUserCredentials = async (props: Record) => { describe("MCPServerView", () => { beforeEach(() => { vi.clearAllMocks(); + sessionStorage.clear(); }); // Name, alias and description each label the header and a Settings row, so @@ -146,6 +149,37 @@ describe("MCPServerView", () => { expect(screen.queryByRole("button", { name: "Edit Settings" })).not.toBeInTheDocument(); }); + it.each([false, true])("keeps config settings read-only with isEditing=%s", async (isEditing) => { + renderView({ is_config: true }, { isEditing }); + + await userEvent.click(screen.getByRole("tab", { name: "Settings" })); + + expect(screen.getByRole("button", { name: "Edit Settings" })).toBeDisabled(); + expect(screen.getByText("Defined in config. Edit your YAML configuration to make changes")).toBeVisible(); + expect(screen.queryByText("edit form")).not.toBeInTheDocument(); + }); + + it.each([true, false])("honors config read-only state on OAuth return: %s", async (isConfig) => { + setSecureItem(EDIT_OAUTH_UI_STATE_KEY, JSON.stringify({ serverId: "srv-1" })); + renderView({ is_config: isConfig }); + + await userEvent.click(screen.getByRole("tab", { name: "Settings" })); + + if (isConfig) { + expect(screen.queryByText("edit form")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Edit Settings" })).toBeDisabled(); + } else { + expect(screen.getByText("edit form")).toBeVisible(); + } + }); + + it("does not open the editor for a view-only admin", async () => { + renderView({}, { isViewOnly: true, isEditing: true }); + await userEvent.click(screen.getByRole("tab", { name: "Settings" })); + expect(screen.getByRole("button", { name: "Edit Settings" })).toBeDisabled(); + expect(screen.queryByText("edit form")).not.toBeInTheDocument(); + }); + it("opens on the tab named by initialTabIndex", async () => { renderView({}, { initialTabIndex: 1 }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx index a346c7d986b..6045be4607d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx @@ -62,7 +62,8 @@ export const MCPServerView: React.FC = ({ }) => { // Open the editing Settings tab on first render when returning from the edit OAuth // redirect, so the "token fetched" feedback shows where the user left off (Settings=2). - const returningFromEditOAuth = isReturningFromEditOAuth(isProxyAdmin, mcpServer.server_id); + const canEdit = isProxyAdmin && !isViewOnly && !mcpServer.is_config; + const returningFromEditOAuth = isReturningFromEditOAuth(canEdit, mcpServer.server_id); const [editing, setEditing] = useState(isEditing || returningFromEditOAuth); const [showFullUrl, setShowFullUrl] = useState(false); const [copiedStates, setCopiedStates] = useState>({}); @@ -224,13 +225,18 @@ export const MCPServerView: React.FC = ({

MCP Server Settings

- {editing ? null : ( - )}
- {editing ? ( + {mcpServer.is_config && ( +

+ Defined in config. Edit your YAML configuration to make changes +

+ )} + {editing && canEdit ? ( { renderWithProviders(); expect(screen.getByRole("radio", { name: /Day-by-day by team and model/i })).toBeChecked(); }); + + it("should offer the per-user scope for teams and call onChange with daily_with_users", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + renderWithProviders(); + + const option = screen.getByRole("radio", { name: /Day-by-day breakdown by team and user/i }); + await user.click(option); + + expect(onChange).toHaveBeenCalledWith("daily_with_users"); + expect(screen.getByText("Daily metrics for each team, split by key owner")).toBeInTheDocument(); + }); + + it("should hide the per-user scope for user exports while keeping the other scopes", () => { + renderWithProviders(); + + expect(screen.queryByRole("radio", { name: /and user/i })).not.toBeInTheDocument(); + expect( + screen.getByRole("radio", { name: /Day-by-day breakdown by user Daily metrics for each user$/i }), + ).toBeInTheDocument(); + expect(screen.getByRole("radio", { name: /Day-by-day breakdown by user and key/i })).toBeInTheDocument(); + expect(screen.getByRole("radio", { name: /Day-by-day by user and model/i })).toBeInTheDocument(); + }); + + it("should offer the per-user scope for tags", () => { + renderWithProviders(); + + expect(screen.getByRole("radio", { name: /Day-by-day breakdown by tag and user/i })).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.tsx index edd055ba7a7..f6fdece83a8 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.tsx +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.tsx @@ -9,7 +9,7 @@ interface ExportTypeSelectorProps { } const ExportTypeSelector: React.FC = ({ value, onChange, entityType }) => { - const scopes: { value: ExportScope; title: string; description: string }[] = [ + const allScopes: { value: ExportScope; title: string; description: string }[] = [ { value: "daily", title: `Day-by-day breakdown by ${entityType}`, @@ -25,7 +25,13 @@ const ExportTypeSelector: React.FC = ({ value, onChange title: `Day-by-day by ${entityType} and model`, description: "Daily metrics split by model", }, + { + value: "daily_with_users", + title: `Day-by-day breakdown by ${entityType} and user`, + description: `Daily metrics for each ${entityType}, split by key owner`, + }, ]; + const scopes = allScopes.filter((scope) => scope.value !== "daily_with_users" || entityType !== "user"); return (
diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/types.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/types.ts index 30714ad632d..15f193ecc3f 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/types.ts +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/types.ts @@ -2,7 +2,7 @@ import type { DateRangePickerValue } from "@/components/shared/date_picker_types import type { Team } from "@/components/key_team_helpers/key_list"; export type ExportFormat = "csv" | "json"; -export type ExportScope = "daily" | "daily_with_keys" | "daily_with_models"; +export type ExportScope = "daily" | "daily_with_keys" | "daily_with_models" | "daily_with_users"; export type EntityType = "tag" | "team" | "organization" | "customer" | "agent" | "user"; export interface EntitySpendData { diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts index 7ed014d43b2..3f9cf58ec20 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts @@ -8,6 +8,7 @@ import { generateDailyData, generateDailyWithKeysData, generateDailyWithModelsData, + generateDailyWithUsersData, generateExportData, generateMetadata, getEntityBreakdown, @@ -151,6 +152,204 @@ describe("EntityUsageExport utils", () => { "team-2": "Team Two", }; + const usersFixture: EntitySpendData = { + results: [ + { + date: "2025-03-01", + breakdown: { + entities: { + "team-1": { + metrics: { + spend: 16.5, + api_requests: 165, + successful_requests: 156, + failed_requests: 9, + total_tokens: 1650, + prompt_tokens: 940, + completion_tokens: 710, + cache_read_input_tokens: 90, + cache_creation_input_tokens: 60, + }, + api_key_breakdown: { + kA: { + metrics: { + spend: 1.1, + api_requests: 11, + successful_requests: 10, + failed_requests: 1, + total_tokens: 110, + prompt_tokens: 60, + completion_tokens: 50, + cache_read_input_tokens: 6, + cache_creation_input_tokens: 4, + }, + metadata: { + team_id: "team-1", + key_alias: "alice-key", + user_id: "u1", + user_email: "a@x", + }, + }, + kB: { + metrics: { + spend: 2.2, + api_requests: 22, + successful_requests: 20, + failed_requests: 2, + total_tokens: 220, + prompt_tokens: 130, + completion_tokens: 90, + cache_read_input_tokens: 12, + cache_creation_input_tokens: 8, + }, + metadata: { + team_id: "team-1", + user_id: "u1", + user_email: "a@x", + }, + }, + kC: { + metrics: { + spend: 3.3, + api_requests: 33, + successful_requests: 31, + failed_requests: 2, + total_tokens: 330, + prompt_tokens: 190, + completion_tokens: 140, + cache_read_input_tokens: 18, + cache_creation_input_tokens: 12, + }, + metadata: { + team_id: "team-1", + user_id: "u2", + user_email: null, + }, + }, + kD: { + metrics: { + spend: 4.4, + api_requests: 44, + successful_requests: 42, + failed_requests: 2, + total_tokens: 440, + prompt_tokens: 250, + completion_tokens: 190, + cache_read_input_tokens: 24, + cache_creation_input_tokens: 16, + }, + metadata: { + team_id: "team-1", + user_id: null, + }, + }, + kE: { + metrics: { + spend: 5.5, + api_requests: 55, + successful_requests: 53, + failed_requests: 2, + total_tokens: 550, + prompt_tokens: 310, + completion_tokens: 240, + cache_read_input_tokens: 30, + cache_creation_input_tokens: 20, + }, + metadata: { + team_id: "team-1", + user_id: "u3", + key_exists: false, + }, + }, + }, + }, + "team-2": { + metrics: { + spend: 6.6, + api_requests: 66, + successful_requests: 64, + failed_requests: 2, + total_tokens: 660, + prompt_tokens: 370, + completion_tokens: 290, + cache_read_input_tokens: 36, + cache_creation_input_tokens: 24, + }, + api_key_breakdown: { + kF: { + metrics: { + spend: 6.6, + api_requests: 66, + successful_requests: 64, + failed_requests: 2, + total_tokens: 660, + prompt_tokens: 370, + completion_tokens: 290, + cache_read_input_tokens: 36, + cache_creation_input_tokens: 24, + }, + metadata: { + team_id: "team-2", + user_id: "u1", + user_email: "a@x", + }, + }, + }, + }, + }, + }, + }, + { + date: "2025-03-02", + breakdown: { + entities: { + "team-1": { + metrics: { + spend: 7.7, + api_requests: 77, + successful_requests: 75, + failed_requests: 2, + total_tokens: 770, + prompt_tokens: 430, + completion_tokens: 340, + cache_read_input_tokens: 42, + cache_creation_input_tokens: 28, + }, + api_key_breakdown: { + kA: { + metrics: { + spend: 7.7, + api_requests: 77, + successful_requests: 75, + failed_requests: 2, + total_tokens: 770, + prompt_tokens: 430, + completion_tokens: 340, + cache_read_input_tokens: 42, + cache_creation_input_tokens: 28, + }, + metadata: { + team_id: "team-1", + key_alias: "alice-key", + user_id: "u1", + user_email: "a@x", + }, + }, + }, + }, + }, + }, + }, + ], + metadata: { + total_spend: 30.8, + total_api_requests: 308, + total_successful_requests: 295, + total_failed_requests: 13, + total_tokens: 3080, + }, + }; + beforeEach(() => { vi.clearAllMocks(); }); @@ -1056,6 +1255,27 @@ describe("EntityUsageExport utils", () => { expect(keyIds).toContain("key1"); expect(keyIds).toContain("key2"); }); + + it("should emit key owner columns right after Key ID", () => { + const result = generateDailyWithKeysData(usersFixture, "Team"); + + const columnNames = Object.keys(result[0]); + expect(columnNames[columnNames.indexOf("Key ID") + 1]).toBe("User ID"); + expect(columnNames[columnNames.indexOf("User ID") + 1]).toBe("User Email"); + + const kARow = result.find((r) => r["Key ID"] === "kA" && r.Date === "2025-03-01"); + expect(kARow?.["User ID"]).toBe("u1"); + expect(kARow?.["User Email"]).toBe("a@x"); + expect(kARow?.["Key Alias"]).toBe("alice-key"); + + const kCRow = result.find((r) => r["Key ID"] === "kC"); + expect(kCRow?.["User ID"]).toBe("u2"); + expect(kCRow?.["User Email"]).toBe("-"); + + const kDRow = result.find((r) => r["Key ID"] === "kD"); + expect(kDRow?.["User ID"]).toBe("-"); + expect(kDRow?.["User Email"]).toBe("-"); + }); }); describe("generateDailyWithModelsData", () => { @@ -2010,6 +2230,21 @@ describe("EntityUsageExport utils", () => { window.Blob = originalBlob; }); + + it("should generate the daily_with_users filename and include User ID in the rows", () => { + const anchorElement = document.createElement("a"); + vi.spyOn(document, "createElement").mockReturnValue(anchorElement); + vi.useFakeTimers(); + vi.setSystemTime(new Date("2025-03-01T12:00:00Z")); + + handleExportCSV(usersFixture, "daily_with_users", "Team", "team", mockTeamAliasMap); + vi.useRealTimers(); + + expect(anchorElement.download).toBe("team_usage_daily_with_users_2025-03-01.csv"); + + const unparsedRows = vi.mocked(Papa.unparse).mock.calls[0][0] as Record[]; + expect(unparsedRows[0]).toHaveProperty("User ID"); + }); }); describe("handleExportJSON", () => { @@ -2462,4 +2697,312 @@ describe("EntityUsageExport utils", () => { expect(result.find((r) => r["User ID"] === "user-b")?.["User"]).toBe("Grace"); }); }); + + describe("generateDailyWithUsersData", () => { + it("should reconcile spend with daily_with_keys and daily per date and team", () => { + const byUser = generateDailyWithUsersData(usersFixture, "Team"); + const byKey = generateDailyWithKeysData(usersFixture, "Team"); + const daily = generateDailyData(usersFixture, "Team"); + + expect(byUser.length).toBeGreaterThan(0); + expect(byKey.length).toBeGreaterThan(0); + expect(daily.length).toBeGreaterThan(0); + + const sumSpend = (rows: any[]): Record => { + const totals: Record = {}; + rows.forEach((r) => { + const bucket = `${r.Date}|${r["Team ID"]}`; + totals[bucket] = (totals[bucket] || 0) + Number(r["Spend ($)"]); + }); + return totals; + }; + + const userTotals = sumSpend(byUser); + const keyTotals = sumSpend(byKey); + + daily.forEach((row) => { + const bucket = `${row.Date}|${row["Team ID"]}`; + expect(userTotals[bucket]).toBeCloseTo(Number(row["Spend ($)"]), 4); + expect(keyTotals[bucket]).toBeCloseTo(Number(row["Spend ($)"]), 4); + }); + }); + + it("should roll multiple keys owned by one user in a team into a single row", () => { + const rows = generateDailyWithUsersData(usersFixture, "Team"); + const matches = rows.filter((r) => r.Date === "2025-03-01" && r["Team ID"] === "team-1" && r["User ID"] === "u1"); + + expect(matches).toHaveLength(1); + const row = matches[0]; + expect(row.Keys).toBe(2); + expect(row["User Email"]).toBe("a@x"); + expect(row["Spend ($)"]).toBe("3.3000"); + expect(row.Requests).toBe(33); + expect(row["Successful Requests"]).toBe(30); + expect(row["Failed Requests"]).toBe(3); + expect(row["Total Tokens"]).toBe(330); + expect(row["Prompt Tokens"]).toBe(190); + expect(row["Completion Tokens"]).toBe(140); + expect(row["Cache Read Input Tokens"]).toBe(18); + expect(row["Cache Creation Input Tokens"]).toBe(12); + }); + + it("should bucket keys with no owner into an Unassigned row without dropping spend", () => { + const rows = generateDailyWithUsersData(usersFixture, "Team"); + const row = rows.find( + (r) => r.Date === "2025-03-01" && r["Team ID"] === "team-1" && r["User ID"] === "Unassigned", + ); + + expect(row).toBeDefined(); + expect(row?.["User Email"]).toBe("-"); + expect(Number(row?.["Spend ($)"])).toBeCloseTo(4.4, 4); + }); + + it("should keep different users in the same team as separate rows", () => { + const rows = generateDailyWithUsersData(usersFixture, "Team"); + const teamRows = rows.filter((r) => r.Date === "2025-03-01" && r["Team ID"] === "team-1"); + + const u1Rows = teamRows.filter((r) => r["User ID"] === "u1"); + const u2Rows = teamRows.filter((r) => r["User ID"] === "u2"); + expect(u1Rows).toHaveLength(1); + expect(u2Rows).toHaveLength(1); + expect(Number(u2Rows[0]["Spend ($)"])).toBeCloseTo(3.3, 4); + }); + + it("should keep the same user in different teams as separate rows", () => { + const rows = generateDailyWithUsersData(usersFixture, "Team"); + const u1Rows = rows.filter((r) => r.Date === "2025-03-01" && r["User ID"] === "u1"); + + expect(u1Rows).toHaveLength(2); + const team1Row = u1Rows.find((r) => r["Team ID"] === "team-1"); + const team2Row = u1Rows.find((r) => r["Team ID"] === "team-2"); + expect(team1Row?.Keys).toBe(2); + expect(team2Row?.Keys).toBe(1); + expect(Number(team2Row?.["Spend ($)"])).toBeCloseTo(6.6, 4); + }); + + it("should show a dash email when the user has none", () => { + const rows = generateDailyWithUsersData(usersFixture, "Team"); + const row = rows.find((r) => r.Date === "2025-03-01" && r["Team ID"] === "team-1" && r["User ID"] === "u3"); + + expect(row).toBeDefined(); + expect(row?.["User Email"]).toBe("-"); + }); + + it("should still attribute a deleted key to its user", () => { + const rows = generateDailyWithUsersData(usersFixture, "Team"); + const row = rows.find((r) => r.Date === "2025-03-01" && r["Team ID"] === "team-1" && r["User ID"] === "u3"); + + expect(row).toBeDefined(); + expect(Number(row?.["Spend ($)"])).toBeCloseTo(5.5, 4); + expect(row?.Requests).toBe(55); + }); + + it("should group rows under team_id on the aggregated endpoint shape", () => { + const aggregatedFixture: EntitySpendData = { + results: [ + { + date: "2025-03-01", + breakdown: { + entities: {}, + api_keys: { + kA: { + metrics: { + spend: 1.1, + api_requests: 11, + successful_requests: 10, + failed_requests: 1, + total_tokens: 110, + prompt_tokens: 60, + completion_tokens: 50, + cache_read_input_tokens: 6, + cache_creation_input_tokens: 4, + }, + metadata: { team_id: "team-1", user_id: "u1", user_email: "a@x" }, + }, + kF: { + metrics: { + spend: 6.6, + api_requests: 66, + successful_requests: 64, + failed_requests: 2, + total_tokens: 660, + prompt_tokens: 370, + completion_tokens: 290, + cache_read_input_tokens: 36, + cache_creation_input_tokens: 24, + }, + metadata: { team_id: "team-2", user_id: "u2" }, + }, + }, + }, + }, + ], + metadata: usersFixture.metadata, + }; + + const rows = generateDailyWithUsersData(aggregatedFixture, "Team"); + + expect(rows).toHaveLength(2); + const team1Row = rows.find((r) => r["Team ID"] === "team-1"); + const team2Row = rows.find((r) => r["Team ID"] === "team-2"); + expect(team1Row?.["User ID"]).toBe("u1"); + expect(team2Row?.["User ID"]).toBe("u2"); + expect(Number(team1Row?.["Spend ($)"])).toBeCloseTo(1.1, 4); + expect(Number(team2Row?.["Spend ($)"])).toBeCloseTo(6.6, 4); + }); + + it("should emit the exact column order and sort by date ascending", () => { + const rows = generateDailyWithUsersData(usersFixture, "Team"); + + expect(Object.keys(rows[0])).toEqual([ + "Date", + "Team", + "Team ID", + "User ID", + "User Email", + "Keys", + "Spend ($)", + "Requests", + "Successful Requests", + "Failed Requests", + "Total Tokens", + "Prompt Tokens", + "Completion Tokens", + "Cache Read Input Tokens", + "Cache Creation Input Tokens", + ]); + + const dates = rows.map((r) => new Date(r.Date).getTime()); + for (let i = 0; i < dates.length - 1; i++) { + expect(dates[i]).toBeLessThanOrEqual(dates[i + 1]); + } + }); + + it("should dispatch daily_with_users through generateExportData", () => { + expect(generateExportData(usersFixture, "daily_with_users", "Team")).toEqual( + generateDailyWithUsersData(usersFixture, "Team"), + ); + }); + + it("should keep owners separate when entity and user ids contain underscores", () => { + const collisionFixture: EntitySpendData = { + results: [ + { + date: "2025-03-01", + breakdown: { + entities: { + team_1: { + metrics: { spend: 1, api_requests: 1, total_tokens: 10 }, + api_key_breakdown: { + kX: { + metrics: { spend: 1, api_requests: 1, total_tokens: 10 }, + metadata: { team_id: "team_1", user_id: "u1" }, + }, + }, + }, + team: { + metrics: { spend: 2, api_requests: 2, total_tokens: 20 }, + api_key_breakdown: { + kY: { + metrics: { spend: 2, api_requests: 2, total_tokens: 20 }, + metadata: { team_id: "team", user_id: "1_u1" }, + }, + }, + }, + }, + }, + }, + ], + metadata: usersFixture.metadata, + }; + + const rows = generateDailyWithUsersData(collisionFixture, "Team"); + + expect(rows).toHaveLength(2); + const team1Row = rows.find((r) => r["Team ID"] === "team_1"); + expect(team1Row?.["User ID"]).toBe("u1"); + expect(team1Row?.Keys).toBe(1); + expect(team1Row?.["Spend ($)"]).toBe("1.0000"); + const teamRow = rows.find((r) => r["Team ID"] === "team"); + expect(teamRow?.["User ID"]).toBe("1_u1"); + expect(teamRow?.Keys).toBe(1); + expect(teamRow?.["Spend ($)"]).toBe("2.0000"); + }); + + it("should leave daily and daily_with_models output without user columns", () => { + const daily = generateDailyData(usersFixture, "Team"); + expect(daily[0]).not.toHaveProperty("User ID"); + + const modelsFixture: EntitySpendData = { + results: [ + { + date: "2025-03-01", + breakdown: { + entities: { + "team-1": { + metrics: { + spend: 1.1, + api_requests: 11, + successful_requests: 10, + failed_requests: 1, + total_tokens: 110, + prompt_tokens: 60, + completion_tokens: 50, + }, + api_key_breakdown: { + kA: { + metrics: { + spend: 1.1, + api_requests: 11, + successful_requests: 10, + failed_requests: 1, + total_tokens: 110, + }, + metadata: { team_id: "team-1", user_id: "u1", user_email: "a@x" }, + }, + }, + }, + }, + models: { + "gpt-4o": { + metrics: { spend: 1.1, api_requests: 11, total_tokens: 110 }, + api_key_breakdown: { + kA: { + metrics: { + spend: 1.1, + api_requests: 11, + successful_requests: 10, + failed_requests: 1, + total_tokens: 110, + }, + metadata: {}, + }, + }, + }, + }, + }, + }, + ], + metadata: usersFixture.metadata, + }; + + const modelRows = generateDailyWithModelsData(modelsFixture, "Team"); + expect(modelRows).toHaveLength(1); + expect(Object.keys(modelRows[0])).toEqual([ + "Date", + "Team", + "Team ID", + "Model", + "Spend ($)", + "Requests", + "Successful", + "Failed", + "Total Tokens", + "Prompt Tokens", + "Completion Tokens", + "Cache Read Input Tokens", + "Cache Creation Input Tokens", + ]); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts index 8fd75134bcc..95ce584cc89 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts @@ -166,6 +166,8 @@ export const generateDailyWithKeysData = ( entityAlias: string; keyId: string; keyAlias: string | null; + userId: string | null; + userEmail: string | null; metrics: { spend: number; api_requests: number; @@ -200,6 +202,8 @@ export const generateDailyWithKeysData = ( entityAlias, keyId, keyAlias, + userId: keyData?.metadata?.user_id || null, + userEmail: keyData?.metadata?.user_email || null, metrics: { spend: keyData.metrics?.spend || 0, api_requests: keyData.metrics?.api_requests || 0, @@ -236,6 +240,7 @@ export const generateDailyWithKeysData = ( [`${entityLabel} ID`]: item.entityId, "Key Alias": item.keyAlias || "-", "Key ID": item.keyId, + ...(entityLabel === "User" ? {} : { "User ID": item.userId || "-", "User Email": item.userEmail || "-" }), "Spend ($)": formatNumberWithCommas(item.metrics.spend, 4), Requests: item.metrics.api_requests, "Successful Requests": item.metrics.successful_requests, @@ -250,6 +255,71 @@ export const generateDailyWithKeysData = ( return dailyKeyBreakdown.sort((a, b) => new Date(a.Date).getTime() - new Date(b.Date).getTime()); }; +export const generateDailyWithUsersData = ( + spendData: EntitySpendData, + entityLabel: string, + teamAliasMap: Record = {}, +): any[] => { + const aggregatedData: { + [key: string]: { + Date: string; + entityId: string; + entityAlias: string; + userId: string; + userEmail: string | null; + keyIds: Set; + metrics: Record<(typeof METRIC_KEYS)[number], number>; + }; + } = {}; + + spendData.results.forEach((day) => { + Object.entries(resolveEntities(day.breakdown)).forEach(([entity, data]: [string, any]) => { + const { id: entityId, alias: entityAlias } = resolveEntityDisplay(entity, teamAliasMap, data.metadata); + Object.entries(data.api_key_breakdown || {}).forEach(([keyId, keyData]: [string, any]) => { + const userId = keyData?.metadata?.user_id || "Unassigned"; + const uniqueKey = JSON.stringify([day.date, entityId, userId]); + if (!aggregatedData[uniqueKey]) { + aggregatedData[uniqueKey] = { + Date: day.date, + entityId, + entityAlias, + userId, + userEmail: null, + keyIds: new Set(), + metrics: Object.fromEntries(METRIC_KEYS.map((k) => [k, 0])) as Record<(typeof METRIC_KEYS)[number], number>, + }; + } + const bucket = aggregatedData[uniqueKey]; + bucket.userEmail = bucket.userEmail || keyData?.metadata?.user_email || null; + bucket.keyIds.add(keyId); + for (const k of METRIC_KEYS) { + bucket.metrics[k] += keyData?.metrics?.[k] || 0; + } + }); + }); + }); + + return Object.values(aggregatedData) + .map((item) => ({ + Date: item.Date, + [entityLabel]: item.entityAlias, + [`${entityLabel} ID`]: item.entityId, + "User ID": item.userId, + "User Email": item.userEmail || "-", + Keys: item.keyIds.size, + "Spend ($)": formatNumberWithCommas(item.metrics.spend, 4), + Requests: item.metrics.api_requests, + "Successful Requests": item.metrics.successful_requests, + "Failed Requests": item.metrics.failed_requests, + "Total Tokens": item.metrics.total_tokens, + "Prompt Tokens": item.metrics.prompt_tokens, + "Completion Tokens": item.metrics.completion_tokens, + "Cache Read Input Tokens": item.metrics.cache_read_input_tokens, + "Cache Creation Input Tokens": item.metrics.cache_creation_input_tokens, + })) + .sort((a, b) => new Date(a.Date).getTime() - new Date(b.Date).getTime()); +}; + export const generateDailyWithModelsData = ( spendData: EntitySpendData, entityLabel: string, @@ -340,6 +410,8 @@ export const generateExportData = ( return generateDailyWithKeysData(spendData, entityLabel, teamAliasMap); case "daily_with_models": return generateDailyWithModelsData(spendData, entityLabel, teamAliasMap); + case "daily_with_users": + return generateDailyWithUsersData(spendData, entityLabel, teamAliasMap); default: return generateDailyData(spendData, entityLabel, teamAliasMap); } diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index 4d963a2f603..7ef7f1fcb09 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -58,6 +58,10 @@ vi.mock("@/app/(dashboard)/hooks/keys/useKeyInfo", () => ({ useKeyInfo: vi.fn(), })); +vi.mock("@/app/(dashboard)/hooks/uiSettings/useApplyUserBudgetToTeamKeys", () => ({ + useApplyUserBudgetToTeamKeys: vi.fn(() => false), +})); + vi.mock("@/app/(dashboard)/hooks/useTeams", () => ({ default: vi.fn(), })); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx index 39dd2cc5ab2..ec405ee9cad 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx @@ -3,6 +3,7 @@ import { useKeyInfo } from "@/app/(dashboard)/hooks/keys/useKeyInfo"; import { useKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; import { useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; +import { useApplyUserBudgetToTeamKeys } from "@/app/(dashboard)/hooks/uiSettings/useApplyUserBudgetToTeamKeys"; import { useAllTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; import { @@ -139,10 +140,18 @@ export function VirtualKeysTable({ headerActions }: VirtualKeysTableProps) { const keyList = useMemo(() => keys?.keys ?? [], [keys]); const rowCount = keys?.total_count ?? 0; - const columns = useMemo( - () => getKeyTableColumns({ allTeams, organizations, onSelectKey: (key) => void setSelectedKeyId(key.token) }), - [allTeams, organizations, setSelectedKeyId], + const applyUserBudgetToTeamKeys = useApplyUserBudgetToTeamKeys(); + + const columnDeps = useMemo( + () => ({ + allTeams, + organizations, + onSelectKey: (key: KeyResponse) => void setSelectedKeyId(key.token), + applyUserBudgetToTeamKeys, + }), + [allTeams, organizations, setSelectedKeyId, applyUserBudgetToTeamKeys], ); + const columns = useMemo(() => getKeyTableColumns(columnDeps), [columnDeps]); const selectedKeyFromList = useMemo( () => keyList.find((key) => key.token === selectedKeyId), diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx index c9ee0fa1126..33625c0e1eb 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx @@ -4,7 +4,7 @@ import { Info } from "lucide-react"; import { ColumnDef } from "@tanstack/react-table"; import { DataTableMultiSortHeader, DataTableSortHeader, type DataTableSortField } from "@/components/shared/DataTable"; -import { inheritedBudgetGates } from "@/components/shared/InheritedBudgetHint"; +import { inheritedBudgetGates, keyOwnerBudgetSource } from "@/components/shared/InheritedBudgetHint"; import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card"; import { Skeleton } from "@/components/ui/skeleton"; import { @@ -86,12 +86,14 @@ interface KeyTableColumnsDeps { allTeams: Team[]; organizations: Organization[]; onSelectKey: (key: KeyResponse) => void; + applyUserBudgetToTeamKeys: boolean; } export const getKeyTableColumns = ({ allTeams, organizations, onSelectKey, + applyUserBudgetToTeamKeys, }: KeyTableColumnsDeps): ColumnDef[] => [ { id: "key_alias", @@ -277,7 +279,11 @@ export const getKeyTableColumns = ({ ); }, diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx index 71f8bce76b5..44adecb9e1d 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx @@ -141,7 +141,7 @@ const ComplexityRouterAdvancedSections: React.FCAdvanced: Reminder Markers, + label: Advanced: Ignore Custom Tags, children: , }, { diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 56b37b92af3..756e505997c 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -101,7 +101,7 @@ describe("ComplexityRouterConfig", () => { expect(screen.getByText("Advanced: Heuristic Keyword Overrides")).toBeInTheDocument(); expect(screen.getByText("Advanced: Housekeeping Routing")).toBeInTheDocument(); - expect(screen.getByText("Advanced: Reminder Markers")).toBeInTheDocument(); + expect(screen.getByText("Advanced: Ignore Custom Tags")).toBeInTheDocument(); const capabilityValue = { ...defaultValue, classifier_type: "capability" as const }; rerender(); @@ -128,7 +128,7 @@ describe("ComplexityRouterConfig", () => { renderWithProviders( , ); - fireEvent.click(screen.getByText("Advanced: Reminder Markers")); + fireEvent.click(screen.getByText("Advanced: Ignore Custom Tags")); const validation = screen.queryByText(/needs both/i); if (showValidationErrors) { expect(validation).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx b/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx index c7f9ee48e0b..88901bb8645 100644 --- a/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ReminderMarkers.tsx @@ -26,31 +26,32 @@ const ReminderMarkers: React.FC<{ return (

- Delimiter pairs that wrap harness-injected reminder blocks, which are stripped before classification. Setting - any pair replaces the built-in pairs, so list every pair your harness emits. Matching is case-insensitive and - values are saved lowercased. + Auto Router already ignores built-in reminder tags, such as <system-reminder>, when choosing + a model. Leave this empty to keep the defaults. Add custom opening and closing tags to ignore the text between + them when routing. Custom pairs replace the defaults, so include any built-in pairs you still need. The selected + model still receives the full message. Matching is case-insensitive.

{markers.map((marker, index) => (
update(index, { open: event.target.value })} />
update(index, { close: event.target.value })} /> @@ -58,7 +59,7 @@ const ReminderMarkers: React.FC<{ {showValidationErrors && error &&

{error}

}
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 9b4f1980f62..13df40464f0 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -414,8 +414,8 @@ export const getReminderMarkersError = (pairs: ReminderMarkerPair[] | undefined) for (const [index, pair] of (pairs ?? []).entries()) { const open = pair.open.trim().toLowerCase(); const close = pair.close.trim().toLowerCase(); - if (!open || !close) return `Reminder marker pair ${index + 1} needs both an opening and a closing delimiter`; - if (open === close) return `Reminder marker pair ${index + 1} must use different opening and closing delimiters`; + if (!open || !close) return `Tag pair ${index + 1} needs both an opening and a closing tag`; + if (open === close) return `Tag pair ${index + 1} must use different opening and closing tags`; } return null; }; diff --git a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx index b43a85d18c5..f5138d55b5d 100644 --- a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx @@ -30,6 +30,8 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ dynamic_params: { arize_api_key: "password", arize_space_id: "password", + arize_success_sampling_rate: "number", + arize_error_sampling_rate: "number", }, description: "Arize Logging Integration", }, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx index aa6cf92ceb9..6d66152cee5 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx @@ -377,9 +377,9 @@ describe("EditAutoRouterModal advanced field round trips", () => { expect(screen.getByRole("switch", { name: "Route housekeeping calls to the cheapest tier" })).not.toBeChecked(); expect(screen.getByRole("combobox", { name: "e.g., conversation title" })).toHaveValue(""); - await user.click(screen.getByText("Advanced: Reminder Markers")); - expect(screen.getByLabelText("Opening delimiter")).toHaveValue(""); - expect(screen.getByLabelText("Closing delimiter")).toHaveValue(""); + await user.click(screen.getByText("Advanced: Ignore Custom Tags")); + expect(screen.getByLabelText("Opening tag")).toHaveValue(""); + expect(screen.getByLabelText("Closing tag")).toHaveValue(""); await user.click(screen.getByText("Advanced: Response Format")); const maxTokensSwitch = screen.getByRole("switch", { name: "Cap max_tokens at the tier model's output ceiling" }); diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx index ad357f5925d..8d39a0fd22b 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx @@ -115,8 +115,10 @@ export interface KeyResponse { next_rotation_at?: string; user?: { user_id: string; - user_email: string; + user_email: string | null; user_alias: string | null; + max_budget?: number | null; + budget_duration?: string | null; }; created_by_user?: { user_id: string; diff --git a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx index f009429693b..47df369fb8a 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx @@ -407,6 +407,7 @@ export interface MCPToolsViewerProps { export interface MCPServer { server_id: string; + is_config?: boolean; server_name?: string | null; alias?: string | null; description?: string | null; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 82399792674..d358d23408c 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2003,6 +2003,7 @@ interface UiSpendLogsParams { end_user?: string; status_filter?: string; cache_hit_filter?: string; + span_type?: string; /** Filter by model name (e.g. "gpt-4") */ model?: string; /** Filter by model ID (litellm model deployment id) */ diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.test.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.integration.test.tsx similarity index 67% rename from ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.test.tsx rename to ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.integration.test.tsx index e376c551923..4a9a05c1023 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.test.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.integration.test.tsx @@ -1,4 +1,5 @@ import userEvent from "@testing-library/user-event"; +import { fireEvent } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; import { renderWithProviders, screen } from "@/../tests/test-utils"; @@ -6,7 +7,7 @@ import { renderWithProviders, screen } from "@/../tests/test-utils"; import RoutingGroupModal from "./RoutingGroupModal"; import type { RoutingGroup } from "./types"; -const STRATEGIES = ["simple-shuffle", "latency-based-routing", "usage-based-routing"]; +const STRATEGIES = ["simple-shuffle", "latency-based-routing", "usage-based-routing", "priority"]; const MODEL_OPTIONS = ["gpt-4o", "claude-sonnet", "gemini-pro"]; const STRATEGY_DESCRIPTIONS = { "simple-shuffle": "Spreads requests evenly across the group." }; @@ -40,6 +41,14 @@ const STORED_GROUP_NULL_ARGS: RoutingGroup = { routing_strategy_args: null, }; +const STORED_PRIORITY_GROUP: RoutingGroup = { + group_name: "preferred-chat", + models: ["gpt-4o", "claude-sonnet"], + routing_strategy: "priority", + routing_strategy_args: null, + model_priorities: { "gpt-4o": 3, "claude-sonnet": 7 }, +}; + const EXPECTED_NULL_ARGS_PAYLOAD: RoutingGroup = { group_name: "already-taken", models: ["gpt-4o"], @@ -98,6 +107,138 @@ const save = async (user: ReturnType, name: string) => await user.click(screen.getByRole("button", { name })); describe("RoutingGroupModal", () => { + it("creates a priority group with editable defaults and members already used by another group", async () => { + const user = userEvent.setup(); + const { onSubmit } = renderModal({ groupNameByModel: { "gpt-4o": "legacy-group" } }); + + fireEvent.change(screen.getByLabelText("Group Name"), { target: { value: "preferred-chat" } }); + await pickModels(user, "gpt-4o", "claude-sonnet"); + await pickStrategy(user, "Priority"); + + expect(screen.getByLabelText("Priority for gpt-4o")).toHaveValue("1"); + expect(screen.getByLabelText("Priority for claude-sonnet")).toHaveValue("2"); + fireEvent.change(screen.getByLabelText("Priority for claude-sonnet"), { target: { value: "1" } }); + await save(user, "Create Group"); + + const expected: RoutingGroup = { + group_name: "preferred-chat", + models: ["gpt-4o", "claude-sonnet"], + routing_strategy: "priority", + routing_strategy_args: null, + model_priorities: { "gpt-4o": 1, "claude-sonnet": 1 }, + }; + expect(onSubmit).toHaveBeenCalledWith(expected); + }); + + it("preserves explicit priorities through edit for model names that are dictionary keys", async () => { + const user = userEvent.setup(); + const stored: RoutingGroup = { + group_name: "preferred-chat", + models: ["provider/model.v1", "constructor", "__proto__"], + routing_strategy: "priority", + routing_strategy_args: null, + model_priorities: Object.fromEntries([ + ["__proto__", 3], + ["constructor", 8], + ["provider/model.v1", 2], + ]), + }; + const { onSubmit } = renderModal({ mode: "edit", initialValue: stored, modelOptions: stored.models }); + + expect(screen.getByLabelText("Priority for provider/model.v1")).toHaveValue("2"); + expect(screen.getByLabelText("Priority for constructor")).toHaveValue("8"); + await save(user, "Save Changes"); + + expect(onSubmit).toHaveBeenCalledWith(stored); + }); + + it.each([ + { catalog: "omits Priority", strategies: ["simple-shuffle"] }, + { catalog: "includes Priority", strategies: STRATEGIES }, + ])("keeps unsaved priorities when switching away and back while the catalog $catalog", async ({ strategies }) => { + const user = userEvent.setup(); + const { onSubmit } = renderModal({ + mode: "edit", + initialValue: STORED_PRIORITY_GROUP, + availableStrategies: strategies, + }); + + fireEvent.change(screen.getByLabelText("Priority for gpt-4o"), { target: { value: "9" } }); + await pickStrategy(user, "simple-shuffle"); + expect(screen.queryByLabelText("Priority for gpt-4o")).not.toBeInTheDocument(); + await pickStrategy(user, "Priority"); + expect(screen.getByLabelText("Priority for gpt-4o")).toHaveValue("9"); + expect(screen.getByLabelText("Priority for claude-sonnet")).toHaveValue("7"); + await save(user, "Save Changes"); + + expect(onSubmit).toHaveBeenCalledWith({ + ...STORED_PRIORITY_GROUP, + model_priorities: { "gpt-4o": 9, "claude-sonnet": 7 }, + }); + }); + + it("offers only the backend catalog when creating a group", async () => { + const user = userEvent.setup(); + renderModal({ initialValue: STORED_PRIORITY_GROUP, availableStrategies: ["simple-shuffle"] }); + + await user.click(screen.getByLabelText("Routing Strategy")); + + expect(await screen.findByRole("option", { name: "simple-shuffle" })).toBeInTheDocument(); + expect(screen.queryByRole("option", { name: "Priority" })).not.toBeInTheDocument(); + }); + + it("shows invalid stored priorities and lets the user repair them before saving", async () => { + const user = userEvent.setup(); + const { onSubmit } = renderModal({ + mode: "edit", + initialValue: { + group_name: "preferred-chat", + models: ["gpt-4o"], + routing_strategy: "priority", + model_priorities: { "gpt-4o": 0, unused: 9 }, + }, + }); + + expect(screen.getByLabelText("Priority for gpt-4o")).toHaveValue("0"); + expect(screen.getByText("Model is not selected")).toBeInTheDocument(); + await save(user, "Save Changes"); + expect(onSubmit).not.toHaveBeenCalled(); + await user.click(screen.getByRole("button", { name: "Remove priority for unused" })); + await save(user, "Save Changes"); + expect(await screen.findByText("Priorities must be whole numbers from 1 to 9007199254740991")).toBeInTheDocument(); + fireEvent.change(screen.getByLabelText("Priority for gpt-4o"), { target: { value: "4" } }); + await save(user, "Save Changes"); + + const expected: RoutingGroup = { + group_name: "preferred-chat", + models: ["gpt-4o"], + routing_strategy: "priority", + routing_strategy_args: null, + model_priorities: { "gpt-4o": 4 }, + }; + expect(onSubmit).toHaveBeenCalledWith(expected); + }); + + it("restores the legacy ownership restriction when switching away from priority", async () => { + const user = userEvent.setup(); + const { onSubmit } = renderModal({ + mode: "edit", + initialValue: { + group_name: "preferred-chat", + models: ["gpt-4o"], + routing_strategy: "priority", + model_priorities: { "gpt-4o": 1 }, + }, + groupNameByModel: { "gpt-4o": "legacy-group" }, + }); + + await pickStrategy(user, "simple-shuffle"); + await save(user, "Save Changes"); + + expect(await screen.findByText(/Already claimed: gpt-4o/)).toBeInTheDocument(); + expect(onSubmit).not.toHaveBeenCalled(); + }); + it("submits an untouched edit of a group whose stored arguments are null", async () => { const user = userEvent.setup(); const { onSubmit } = renderModal({ mode: "edit", initialValue: STORED_GROUP_NULL_ARGS }); diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.tsx index 1057cc6ca16..01bdbae4bf2 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupModal.tsx @@ -26,6 +26,7 @@ import { STRATEGIES_WITH_ARGS, argsForStrategy, buildRoutingGroupPayload, + prioritiesForModels, toRoutingGroupFormValues, } from "./routingGroupPayload"; import type { RoutingGroup } from "./types"; @@ -65,7 +66,14 @@ const RoutingGroupModal: React.FC = ({ saving, }) => { const modelsAnchor = useComboboxAnchor(); - const strategyItems = availableStrategies.map((strategy) => ({ label: strategy, value: strategy })); + const selectableStrategies = + mode === "edit" && initialValue + ? Array.from(new Set([...availableStrategies, initialValue.routing_strategy])) + : availableStrategies; + const strategyItems = selectableStrategies.map((strategy) => ({ + label: strategy === "priority" ? "Priority" : strategy, + value: strategy, + })); const reservedNames = useMemo(() => { const others = existingGroupNames.filter((n) => n !== initialValue?.group_name); @@ -80,19 +88,18 @@ const RoutingGroupModal: React.FC = ({ .min(1, "Group name is required") .max(GROUP_NAME_MAX_LENGTH, `Must be ${GROUP_NAME_MAX_LENGTH} characters or fewer`) .refine((value) => !reservedNames.has(value.toLowerCase()), "A group with this name already exists"), - models: z - .array(z.string()) - .min(1, "Select at least one model") - .superRefine((models, ctx) => { - const conflict = modelConflictError(models, groupNameByModel); - if (conflict !== null) { - ctx.addIssue({ code: "custom", message: conflict }); - } - }), + models: z.array(z.string()).min(1, "Select at least one model"), routing_strategy: z.string().min(1, "Strategy is required"), routing_strategy_args: z.string(), + model_priorities: z.array(z.object({ model: z.string(), priority: z.string() })), }; - return z.object(shape); + return z.object(shape).superRefine((values, ctx) => { + if (values.routing_strategy === "priority") return; + const conflict = modelConflictError(values.models, groupNameByModel); + if (conflict !== null) { + ctx.addIssue({ code: "custom", message: conflict, path: ["models"] }); + } + }); }, [reservedNames, groupNameByModel]); const form = useZodForm(schema, { defaultValues: toRoutingGroupFormValues(initialValue, availableStrategies) }); @@ -102,11 +109,12 @@ const RoutingGroupModal: React.FC = ({ }, [open, initialValue, availableStrategies, form]); const selectedStrategy = useWatch({ control: form.control, name: "routing_strategy" }); + const selectedModels = useWatch({ control: form.control, name: "models" }); const handleSubmit = async (values: z.infer) => { const payload = buildRoutingGroupPayload(values); if (!payload.ok) { - form.setError("routing_strategy_args", { message: payload.argsError }); + form.setError(payload.field, { message: payload.message }); return; } await onSubmit(payload.group); @@ -126,7 +134,7 @@ const RoutingGroupModal: React.FC = ({ control={form.control} name="group_name" label="Group Name" - description="Use this name as the model in API calls — LiteLLM routes the request to one of the group's models." + description="Use this name as the model in API calls. LiteLLM routes the request to one of the group's models." > {({ ref, ...field }) => } @@ -135,10 +143,22 @@ const RoutingGroupModal: React.FC = ({ control={form.control} name="models" label="Models" - description="Models from your model list that this group routes between. A model can only be in one group." + description={ + selectedStrategy === "priority" + ? "Models from your model list that this group routes between. Models can belong to multiple priority groups." + : "Models from your model list that this group routes between. A model can belong to one non-priority group." + } > {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( - + { + onChange(models); + form.setValue("model_priorities", prioritiesForModels(models, form.getValues("model_priorities"))); + }} + > }> {(selected: string[]) => ( @@ -176,7 +196,11 @@ const RoutingGroupModal: React.FC = ({ control={form.control} name="routing_strategy" label="Routing Strategy" - description={strategyDescriptions[selectedStrategy]} + description={ + selectedStrategy === "priority" + ? "Lower priorities are tried first. Models with the same priority share traffic. Applies only when calling this group." + : strategyDescriptions[selectedStrategy] + } > {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( + onChange( + value.map((item, itemIndex) => + itemIndex === index ? { ...item, priority: event.target.value } : item, + ), + ) + } + /> + {!selectedModels.includes(entry.model) && ( + + )} +
+ ))} +
+ )} + + )} + {STRATEGIES_WITH_ARGS.has(selectedStrategy) && ( = ({ )}

- Models not claimed by an explicit group fall through to the proxy's top-level routing strategy. + {selectedStrategy === "priority" + ? "Direct requests to a member model keep their existing routing behavior." + : "Models outside non-priority groups use the proxy's top-level routing strategy."}

diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupUsagePanel.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupUsagePanel.tsx index fafea569012..85dd67099d6 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupUsagePanel.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupUsagePanel.tsx @@ -14,7 +14,8 @@ interface RoutingGroupUsagePanelProps { baseUrl: string; } -const exampleModel = (group: RoutingGroup): string => group.models[0] ?? ""; +const exampleModel = (group: RoutingGroup): string => + group.routing_strategy === "priority" ? group.group_name : group.models[0] ?? ""; const buildCurlSnippet = (group: RoutingGroup, baseUrl: string): string => `curl -X POST '${baseUrl}/v1/chat/completions' \\ @@ -69,8 +70,17 @@ export function RoutingGroupUsagePanel({ group, baseUrl }: RoutingGroupUsagePane How routing works for this group

- Callers request any model in the group by name; LiteLLM picks a deployment behind the scenes using the{" "} - {formatStrategyLabel(group.routing_strategy)} strategy. + {group.routing_strategy === "priority" ? ( + <> + Request {group.group_name} to try eligible models in + priority order. Direct requests to a member model keep their existing routing behavior. + + ) : ( + <> + Callers request any model in the group by name; LiteLLM picks a deployment behind the scenes using the{" "} + {formatStrategyLabel(group.routing_strategy)} strategy. + + )}

diff --git a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.test.tsx b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.integration.test.tsx similarity index 86% rename from ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.test.tsx rename to ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.integration.test.tsx index 6f14b76e2fd..6664ecab623 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.test.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/RoutingGroupsTable.integration.test.tsx @@ -113,6 +113,22 @@ describe("RoutingGroupsTable", () => { expect(panel?.textContent).toContain("gpt-4o"); }); + it("uses the callable group name in every priority routing example", async () => { + const user = userEvent.setup(); + render(); + expect(within(rowFor("prod-group")).getByText("Priority")).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "prod-group" })); + + expect(screen.getByRole("tabpanel")).toHaveTextContent('"model": "prod-group"'); + await user.click(screen.getByRole("tab", { name: "Python (OpenAI SDK)" })); + expect(screen.getByRole("tabpanel")).toHaveTextContent('model="prod-group"'); + await user.click(screen.getByRole("tab", { name: "JavaScript (OpenAI SDK)" })); + expect(screen.getByRole("tabpanel")).toHaveTextContent('model: "prod-group"'); + expect( + screen.getByText(/Direct requests to a member model keep their existing routing behavior/), + ).toBeInTheDocument(); + }); + it("should expand only the clicked group", async () => { const user = userEvent.setup(); render(); diff --git a/ui/litellm-dashboard/src/components/routing_groups/index.tsx b/ui/litellm-dashboard/src/components/routing_groups/index.tsx index 17329d0b572..ae2d62ae20f 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/index.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/index.tsx @@ -46,6 +46,7 @@ const RoutingGroups: React.FC = () => { const availableStrategies = useMemo(() => { if (data?.availableStrategies?.length) return data.availableStrategies; + if (routerFields?.routing_group_strategies?.length) return routerFields.routing_group_strategies; const fromFields = routerFields?.fields?.find((f) => f.field_name === "routing_strategy")?.options; return fromFields ?? []; }, [data?.availableStrategies, routerFields]); @@ -178,8 +179,17 @@ const RoutingGroups: React.FC = () => { Delete routing group?

- Models in {deletingGroup?.group_name} will fall back to the - proxy's top-level routing strategy. This cannot be undone. + {deletingGroup?.routing_strategy === "priority" ? ( + <> + Calls to {deletingGroup.group_name} will stop working. Direct + requests to its member models keep their existing routing behavior. This cannot be undone. + + ) : ( + <> + Models in {deletingGroup?.group_name} will fall back to the + proxy's top-level routing strategy. This cannot be undone. + + )}