From 96baeb8b04fd5c31a5a6e99dd0cece61e05960bc Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 16 Sep 2026 16:00:07 +0000 Subject: [PATCH] refactor(rust): remove gateway, config, router, realtime, and trace-parity infrastructure Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/ai-gateway-image.yml | 73 -- .github/workflows/test-rust.yml | 5 - litellm-rust/Cargo.lock | 138 ---- litellm-rust/Cargo.toml | 7 - litellm-rust/crates/ai-gateway/Cargo.toml | 56 -- litellm-rust/crates/ai-gateway/Dockerfile | 109 --- .../crates/ai-gateway/Dockerfile.dockerignore | 54 -- litellm-rust/crates/ai-gateway/README.md | 206 ----- litellm-rust/crates/ai-gateway/config.yaml | 13 - litellm-rust/crates/ai-gateway/render.yaml | 35 - .../src/audio_transcription/hooks.rs | 288 ------- .../ai-gateway/src/audio_transcription/mod.rs | 23 - .../src/audio_transcription/prepare.rs | 55 -- .../src/audio_transcription/tests.rs | 53 -- .../src/audio_transcription/types.rs | 47 -- .../crates/ai-gateway/src/auth/mod.rs | 93 --- .../src/bin/trace_parity_gateway.rs | 42 -- litellm-rust/crates/ai-gateway/src/client.rs | 14 - .../crates/ai-gateway/src/constants.rs | 42 -- .../ai-gateway/src/integrations/README.md | 127 ---- .../src/integrations/custom_guardrail/mod.rs | 468 ------------ .../integrations/custom_guardrail/types.rs | 110 --- .../src/integrations/custom_logger/mod.rs | 317 -------- .../src/integrations/custom_logger/types.rs | 194 ----- .../litellm_python_proxy_api/mod.rs | 197 ----- .../litellm_python_proxy_api/types.rs | 72 -- .../crates/ai-gateway/src/integrations/mod.rs | 12 - .../ai-gateway/src/integrations/types.rs | 83 -- .../ai-gateway/src/io/audio_transcription.rs | 1 - litellm-rust/crates/ai-gateway/src/io/mod.rs | 6 - litellm-rust/crates/ai-gateway/src/io/ocr.rs | 1 - .../crates/ai-gateway/src/io/realtime.rs | 418 ---------- .../crates/ai-gateway/src/io/realtime_pool.rs | 712 ------------------ .../crates/ai-gateway/src/io/responses_ws.rs | 485 ------------ litellm-rust/crates/ai-gateway/src/io/tls.rs | 80 -- litellm-rust/crates/ai-gateway/src/lib.rs | 32 - litellm-rust/crates/ai-gateway/src/main.rs | 162 ---- litellm-rust/crates/ai-gateway/src/ocr/mod.rs | 127 ---- .../crates/ai-gateway/src/ocr/types.rs | 23 - .../crates/ai-gateway/src/realtime/mod.rs | 4 - .../ai-gateway/src/realtime/streaming.rs | 414 ---------- .../crates/ai-gateway/src/routes/AGENTS.md | 43 -- .../crates/ai-gateway/src/routes/health.rs | 24 - .../ai-gateway/src/routes/messages/mod.rs | 532 ------------- .../ai-gateway/src/routes/messages/service.rs | 71 -- .../crates/ai-gateway/src/routes/mod.rs | 25 - .../ai-gateway/src/routes/realtime/README.md | 87 --- .../ai-gateway/src/routes/realtime/mod.rs | 166 ---- .../ai-gateway/src/routes/realtime/service.rs | 77 -- .../ai-gateway/src/routes/responses/mod.rs | 348 --------- .../src/routes/responses/service.rs | 156 ---- litellm-rust/crates/ai-gateway/src/state.rs | 21 - .../crates/ai-gateway/src/trace_parity.rs | 100 --- .../tests/crypto_provider_wiring.rs | 53 -- litellm-rust/crates/config/Cargo.toml | 16 - litellm-rust/crates/config/src/error.rs | 11 - litellm-rust/crates/config/src/lib.rs | 7 - litellm-rust/crates/config/src/python.rs | 76 -- litellm-rust/crates/core/Cargo.toml | 4 - .../core/src/audio_transcription/handler.rs | 1 - .../core/src/audio_transcription/mod.rs | 1 - .../core/src/audio_transcription/prepare.rs | 4 +- .../src/audio_transcription/transformation.rs | 1 - litellm-rust/crates/core/src/auth/vertex.rs | 1 - .../core/src/chat_completions/common_utils.rs | 1 - .../core/src/chat_completions/handler.rs | 1 - .../crates/core/src/chat_completions/mod.rs | 1 - .../core/src/chat_completions/prepare.rs | 3 +- litellm-rust/crates/core/src/constants.rs | 2 - litellm-rust/crates/core/src/http_utils.rs | 1 - litellm-rust/crates/core/src/lib.rs | 5 - .../crates/core/src/messages/common_utils.rs | 1 - .../crates/core/src/messages/handler.rs | 1 - litellm-rust/crates/core/src/messages/mod.rs | 1 - .../crates/core/src/messages/prepare.rs | 3 +- .../core/src/messages/transformation.rs | 2 - .../core/src/observability/function_trace.rs | 215 ------ .../crates/core/src/observability/mod.rs | 59 -- .../azure/document_intelligence/mod.rs | 1 - litellm-rust/crates/core/src/ocr/client.rs | 6 - .../src/ocr/codecs/deepseek/transformation.rs | 1 - .../document_intelligence/transformation.rs | 1 - .../src/ocr/codecs/mistral/transformation.rs | 1 - .../src/ocr/codecs/reducto/transformation.rs | 12 - litellm-rust/crates/core/src/ocr/hooks.rs | 12 - litellm-rust/crates/core/src/ocr/prepare.rs | 1 - litellm-rust/crates/core/src/ocr/registry.rs | 2 +- .../chat_completions/transformation.rs | 3 - .../anthropic/messages/transformation.rs | 1 - .../azure_ai/messages/transformation.rs | 1 - .../providers/bedrock/audio_transcription.rs | 3 - .../chat_completions/transformation.rs | 1 - .../custom_llm_provider.rs} | 0 litellm-rust/crates/core/src/providers/mod.rs | 1 + .../crates/core/src/providers/openai/mod.rs | 1 - .../core/src/providers/openai/realtime/mod.rs | 1 - .../openai/realtime/transformation.rs | 189 ----- litellm-rust/crates/core/src/realtime/mod.rs | 2 - .../core/src/realtime/transformation.rs | 22 - .../crates/core/src/realtime/types.rs | 60 -- .../crates/core/src/router/deployment.rs | 44 -- litellm-rust/crates/core/src/router/mod.rs | 93 --- .../crates/core/src/router/strategy/mod.rs | 26 - .../src/router/strategy/simple_shuffle.rs | 47 -- .../crates/core/src/routing_utils/README.md | 7 - .../crates/core/src/routing_utils/mod.rs | 1 - litellm-rust/crates/python-bridge/Cargo.toml | 6 - .../python-bridge/src/function_trace.rs | 38 - litellm-rust/crates/python-bridge/src/lib.rs | 35 - .../src/routes/audio_transcription/mod.rs | 5 - .../src/routes/chat_completions/mod.rs | 5 - .../python-bridge/src/routes/definition.rs | 92 --- .../python-bridge/src/routes/messages/mod.rs | 5 - .../crates/python-bridge/src/routes/mod.rs | 10 - .../python-bridge/src/routes/ocr/mod.rs | 5 - tests/rust-python-harness/AGENTS.md | 19 +- tests/rust-python-harness/cli/__init__.py | 15 +- tests/rust-python-harness/cli/test_cli.py | 29 +- .../shared/native_build.py | 115 --- .../shared/reporting/strategy.py | 8 - .../shared/test_native_build.py | 121 --- .../shared/tracing/native.py | 39 - .../shared/tracing/steps.py | 231 +----- .../shared/tracing/test_steps.py | 148 +--- .../shared/unit_runners/contracts.py | 109 +++ .../strategies/trace_parity/AGENTS.md | 2 +- .../strategies/trace_parity/__init__.py | 28 +- .../trace_parity/gateway/__init__.py | 1 - .../gateway/chat_completions/case.py | 63 -- .../trace_parity/gateway/execution.py | 197 ----- .../trace_parity/gateway/messages/__init__.py | 1 - .../trace_parity/gateway/messages/case.py | 110 --- .../trace_parity/gateway/responses/case.py | 62 -- .../strategies/trace_parity/models.py | 23 +- .../strategies/trace_parity/reporting.py | 50 +- .../strategies/trace_parity/runner.py | 61 +- .../trace_parity/sdk/chat_completions/case.py | 112 +-- .../strategies/trace_parity/sdk/execution.py | 93 +-- .../trace_parity/sdk/messages/case.py | 217 +----- .../strategies/trace_parity/sdk/ocr/case.py | 274 +------ .../trace_parity/sdk/responses/case.py | 116 +-- .../sdk/test_core_scenario_matrix.py | 16 +- .../trace_parity/sdk/transcription/case.py | 44 +- .../strategies/trace_parity/test_reporting.py | 55 +- .../strategies/trace_parity/test_runner.py | 153 +--- .../strategies/unit_tests_mapping/AGENTS.md | 13 - .../strategies/unit_tests_mapping/__init__.py | 48 -- .../unit_tests_mapping/cases/__init__.py | 1 - .../unit_tests_mapping/cases/ocr.py | 422 ----------- .../unit_tests_mapping/contracts.py | 220 ------ .../unit_tests_mapping/mapping_report.py | 109 --- .../unit_tests_mapping/mapping_validator.py | 296 -------- .../strategies/unit_tests_mapping/mappings.py | 11 - .../unit_tests_mapping/reporting.py | 36 - .../strategies/unit_tests_mapping/runner.py | 61 -- .../test_mapping_validator.py | 314 -------- .../unit_tests_mapping/test_reporting.py | 99 --- .../unit_tests_mapping/test_runner.py | 166 ---- .../strategies/unit_tests_parity/__init__.py | 2 +- .../strategies/unit_tests_rust/__init__.py | 2 +- .../rust_bridge/native_route_wheel_test.py | 2 - tests/test_rust_python_harness.py | 75 +- 162 files changed, 334 insertions(+), 12315 deletions(-) delete mode 100644 .github/workflows/ai-gateway-image.yml delete mode 100644 litellm-rust/crates/ai-gateway/Cargo.toml delete mode 100644 litellm-rust/crates/ai-gateway/Dockerfile delete mode 100644 litellm-rust/crates/ai-gateway/Dockerfile.dockerignore delete mode 100644 litellm-rust/crates/ai-gateway/README.md delete mode 100644 litellm-rust/crates/ai-gateway/config.yaml delete mode 100644 litellm-rust/crates/ai-gateway/render.yaml delete mode 100644 litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/audio_transcription/prepare.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/audio_transcription/tests.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/audio_transcription/types.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/auth/mod.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/bin/trace_parity_gateway.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/client.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/constants.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/integrations/README.md delete mode 100644 litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/mod.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/types.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/integrations/custom_logger/types.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/mod.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/types.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/integrations/mod.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/integrations/types.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/io/audio_transcription.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/io/mod.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/io/ocr.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/io/realtime.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/io/responses_ws.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/io/tls.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/lib.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/main.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/ocr/mod.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/ocr/types.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/realtime/mod.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/realtime/streaming.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/routes/AGENTS.md delete mode 100644 litellm-rust/crates/ai-gateway/src/routes/health.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/routes/messages/service.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/routes/mod.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/routes/realtime/README.md delete mode 100644 litellm-rust/crates/ai-gateway/src/routes/realtime/mod.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/routes/responses/mod.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/routes/responses/service.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/state.rs delete mode 100644 litellm-rust/crates/ai-gateway/src/trace_parity.rs delete mode 100644 litellm-rust/crates/ai-gateway/tests/crypto_provider_wiring.rs delete mode 100644 litellm-rust/crates/config/Cargo.toml delete mode 100644 litellm-rust/crates/config/src/error.rs delete mode 100644 litellm-rust/crates/config/src/lib.rs delete mode 100644 litellm-rust/crates/config/src/python.rs delete mode 100644 litellm-rust/crates/core/src/observability/function_trace.rs delete mode 100644 litellm-rust/crates/core/src/observability/mod.rs rename litellm-rust/crates/core/src/{routing_utils/provider.rs => providers/custom_llm_provider.rs} (100%) delete mode 100644 litellm-rust/crates/core/src/providers/openai/realtime/mod.rs delete mode 100644 litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs delete mode 100644 litellm-rust/crates/core/src/realtime/mod.rs delete mode 100644 litellm-rust/crates/core/src/realtime/transformation.rs delete mode 100644 litellm-rust/crates/core/src/realtime/types.rs delete mode 100644 litellm-rust/crates/core/src/router/deployment.rs delete mode 100644 litellm-rust/crates/core/src/router/mod.rs delete mode 100644 litellm-rust/crates/core/src/router/strategy/mod.rs delete mode 100644 litellm-rust/crates/core/src/router/strategy/simple_shuffle.rs delete mode 100644 litellm-rust/crates/core/src/routing_utils/README.md delete mode 100644 litellm-rust/crates/core/src/routing_utils/mod.rs delete mode 100644 litellm-rust/crates/python-bridge/src/function_trace.rs delete mode 100644 tests/rust-python-harness/shared/native_build.py delete mode 100644 tests/rust-python-harness/shared/test_native_build.py delete mode 100644 tests/rust-python-harness/shared/tracing/native.py create mode 100644 tests/rust-python-harness/shared/unit_runners/contracts.py delete mode 100644 tests/rust-python-harness/strategies/trace_parity/gateway/__init__.py delete mode 100644 tests/rust-python-harness/strategies/trace_parity/gateway/chat_completions/case.py delete mode 100644 tests/rust-python-harness/strategies/trace_parity/gateway/execution.py delete mode 100644 tests/rust-python-harness/strategies/trace_parity/gateway/messages/__init__.py delete mode 100644 tests/rust-python-harness/strategies/trace_parity/gateway/messages/case.py delete mode 100644 tests/rust-python-harness/strategies/trace_parity/gateway/responses/case.py delete mode 100644 tests/rust-python-harness/strategies/unit_tests_mapping/AGENTS.md delete mode 100644 tests/rust-python-harness/strategies/unit_tests_mapping/__init__.py delete mode 100644 tests/rust-python-harness/strategies/unit_tests_mapping/cases/__init__.py delete mode 100644 tests/rust-python-harness/strategies/unit_tests_mapping/cases/ocr.py delete mode 100644 tests/rust-python-harness/strategies/unit_tests_mapping/contracts.py delete mode 100644 tests/rust-python-harness/strategies/unit_tests_mapping/mapping_report.py delete mode 100644 tests/rust-python-harness/strategies/unit_tests_mapping/mapping_validator.py delete mode 100644 tests/rust-python-harness/strategies/unit_tests_mapping/mappings.py delete mode 100644 tests/rust-python-harness/strategies/unit_tests_mapping/reporting.py delete mode 100644 tests/rust-python-harness/strategies/unit_tests_mapping/runner.py delete mode 100644 tests/rust-python-harness/strategies/unit_tests_mapping/test_mapping_validator.py delete mode 100644 tests/rust-python-harness/strategies/unit_tests_mapping/test_reporting.py delete mode 100644 tests/rust-python-harness/strategies/unit_tests_mapping/test_runner.py diff --git a/.github/workflows/ai-gateway-image.yml b/.github/workflows/ai-gateway-image.yml deleted file mode 100644 index 3f690f566b0..00000000000 --- a/.github/workflows/ai-gateway-image.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: ai-gateway image - -on: - push: - paths: - - "litellm-rust/**" - - "litellm/**" - - "enterprise/**" - - "litellm-proxy-extras/**" - - "pyproject.toml" - - "rust-toolchain.toml" - - ".github/workflows/ai-gateway-image.yml" - pull_request: - branches: - - main - - litellm_internal_staging - - litellm_oss_staging - - "litellm_**" - paths: - - "litellm-rust/**" - - "litellm/**" - - "enterprise/**" - - "litellm-proxy-extras/**" - - "pyproject.toml" - - "rust-toolchain.toml" - - ".github/workflows/ai-gateway-image.yml" - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - ai-gateway-image: - name: ai-gateway release image - runs-on: ubuntu-latest - timeout-minutes: 60 - permissions: - contents: read - steps: - - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - name: Build the release image - run: docker build -f litellm-rust/crates/ai-gateway/Dockerfile -t litellm-ai-gateway:${{ github.sha }} . - - name: Start the gateway and wait for readiness - env: - IMAGE: litellm-ai-gateway:${{ github.sha }} - run: | - docker run -d --name ai-gateway -p 4001:4001 \ - -e LITELLM_MASTER_KEY=sk-ci-not-a-real-key \ - -e OPENAI_API_KEY=sk-ci-not-a-real-key \ - "$IMAGE" - for _ in $(seq 1 60); do - if curl -fsS http://127.0.0.1:4001/health/readiness; then - echo "gateway is serving readiness" - exit 0 - fi - sleep 2 - done - echo "gateway never became ready" >&2 - docker logs ai-gateway >&2 - exit 1 - - name: Assert the gateway loaded the baked config - run: | - docker logs ai-gateway 2>&1 | tee gateway.log - grep 'via python config reader' gateway.log - - name: Stop the gateway - if: always() - run: docker rm -f ai-gateway || true diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 17b6481a2bf..87a31d3793c 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -97,8 +97,6 @@ jobs: - run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings - - run: cargo clippy -p litellm-ai-gateway --all-targets --all-features --locked -- -D warnings - rust-test: runs-on: ubuntu-latest timeout-minutes: 30 @@ -134,9 +132,6 @@ jobs: - run: cargo test -p litellm-core --features bedrock-auth --locked working-directory: litellm-rust - - run: cargo test -p litellm-ai-gateway --features server --locked - working-directory: litellm-rust - - run: uv build --wheel --out-dir dist - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 7e3d25e9c5d..500a12035cb 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -462,64 +462,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "axum" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" -dependencies = [ - "async-trait", - "axum-core", - "base64 0.22.1", - "bytes", - "futures-util", - "http 1.4.2", - "http-body 1.1.0", - "http-body-util", - "hyper 1.10.1", - "hyper-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sha1", - "sync_wrapper", - "tokio", - "tokio-tungstenite", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 1.4.2", - "http-body 1.1.0", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "azure_core" version = "1.1.0" @@ -1582,7 +1524,6 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "httparse", - "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1890,51 +1831,12 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - [[package]] name = "libc" version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" -[[package]] -name = "litellm-ai-gateway" -version = "0.1.0" -dependencies = [ - "axum", - "base64 0.22.1", - "futures-channel", - "futures-util", - "litellm-config", - "litellm-core", - "reqwest 0.12.28", - "rustls 0.23.42", - "rustls-native-certs", - "serde", - "serde_json", - "sha2 0.10.9", - "subtle", - "tokio", - "tokio-tungstenite", - "tower", - "tracing", -] - -[[package]] -name = "litellm-config" -version = "0.1.0" -dependencies = [ - "litellm-core", - "pyo3", - "serde_json", - "thiserror 2.0.19", -] - [[package]] name = "litellm-core" version = "0.1.0" @@ -1968,8 +1870,6 @@ dependencies = [ "thiserror 2.0.19", "tokio", "tokio-tungstenite", - "tracing", - "tracing-subscriber", "url", "veil", ] @@ -1990,7 +1890,6 @@ dependencies = [ "serde_json", "tokio", "tokio-tungstenite", - "tracing", ] [[package]] @@ -2065,12 +1964,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - [[package]] name = "memchr" version = "2.8.3" @@ -3150,15 +3043,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - [[package]] name = "shlex" version = "2.0.1" @@ -3380,15 +3264,6 @@ dependencies = [ "syn 3.0.0", ] -[[package]] -name = "thread_local" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" -dependencies = [ - "cfg-if", -] - [[package]] name = "time" version = "0.3.53" @@ -3606,7 +3481,6 @@ dependencies = [ "tokio", "tower-layer", "tower-service", - "tracing", ] [[package]] @@ -3650,7 +3524,6 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3686,17 +3559,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tracing-subscriber" -version = "0.3.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" -dependencies = [ - "sharded-slab", - "thread_local", - "tracing-core", -] - [[package]] name = "try-lock" version = "0.2.5" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 5c72c86d6ef..9212b2048d0 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -2,8 +2,6 @@ members = [ "crates/core", "crates/token-counter", - "crates/config", - "crates/ai-gateway", "crates/python-interop", "crates/python-bridge", ] @@ -17,14 +15,9 @@ repository = "https://github.com/BerriAI/litellm" [workspace.dependencies] bytes = "1" -tracing = "0.1" -tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] } litellm-core = { path = "crates/core" } litellm-token-counter = { path = "crates/token-counter" } -litellm-config = { path = "crates/config" } -litellm-ai-gateway = { path = "crates/ai-gateway", default-features = false } litellm-python-interop = { path = "crates/python-interop" } -axum = "0.7" pyo3 = "0.29.2" pyo3-async-runtimes = { version = "0.29.0", features = ["tokio-runtime"] } pythonize = "0.29.0" diff --git a/litellm-rust/crates/ai-gateway/Cargo.toml b/litellm-rust/crates/ai-gateway/Cargo.toml deleted file mode 100644 index dfa61226d4e..00000000000 --- a/litellm-rust/crates/ai-gateway/Cargo.toml +++ /dev/null @@ -1,56 +0,0 @@ -[package] -name = "litellm-ai-gateway" -version = "0.1.0" -edition.workspace = true -license.workspace = true -repository.workspace = true - -[lib] -name = "litellm_ai_gateway" - -[[bin]] -name = "litellm-ai-gateway" -path = "src/main.rs" -required-features = ["server"] - -[[bin]] -name = "trace-parity-gateway" -path = "src/bin/trace_parity_gateway.rs" -required-features = ["trace-parity"] - -[dependencies] -tracing.workspace = true -litellm-core = { workspace = true, features = ["bedrock-auth"] } -litellm-config.workspace = true -# reqwest (rustls + json) is used by io/ocr and ships realtime logs to the -# Python proxy callbacks API. -reqwest.workspace = true -# rustls and its root store are direct dependencies so `io::tls` can build the -# one TLS config the outbound dials use; see that module for why it has to. -rustls.workspace = true -rustls-native-certs.workspace = true -# `sync` powers the bounded mpsc channel the realtime logger drains. -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "time", "sync"] } -tokio-tungstenite.workspace = true -futures-util.workspace = true -serde_json.workspace = true -base64.workspace = true -axum = { workspace = true, features = ["ws"], optional = true } -serde.workspace = true -subtle = { workspace = true, optional = true } -# sha2 hashes the master key into user_api_key_hash (matches the proxy's -# SHA-256 hash_token) so the plaintext credential never enters a log payload. -sha2 = { workspace = true, optional = true } -tower = { version = "0.5.3", features = ["util"], optional = true } - -[features] -default = [] -server = ["dep:axum", "dep:subtle", "dep:sha2"] -# Build the gateway's config from the proxy YAML via an embedded Python -# interpreter (links libpython; requires `litellm` importable at runtime). -python-config = ["litellm-config/python"] -trace-parity = ["server", "dep:tower", "litellm-core/observability"] - -[dev-dependencies] -futures-channel = "0.3" -tower = { version = "0.5.3", features = ["util"] } diff --git a/litellm-rust/crates/ai-gateway/Dockerfile b/litellm-rust/crates/ai-gateway/Dockerfile deleted file mode 100644 index 72ac25ce1d6..00000000000 --- a/litellm-rust/crates/ai-gateway/Dockerfile +++ /dev/null @@ -1,109 +0,0 @@ -# Multi-stage build for the LiteLLM Rust AI Gateway (realtime WebSocket proxy). -# -# Build context is the **repo root** so we can install `litellm` from this repo's -# source (the gateway loads its model_list via litellm.proxy.read_model_list, -# which is not in any PyPI release yet) AND build the rust workspace under -# litellm-rust/. -# -# docker build -f litellm-rust/crates/ai-gateway/Dockerfile -t litellm-ai-gateway . -# -# No secrets live in this file. Runtime config (LITELLM_MASTER_KEY, -# OPENAI_API_KEY referenced by config.yaml, etc.) is injected as environment -# variables at deploy time. - -# ---- Chef ------------------------------------------------------------------- -# cargo-chef caches the dependency build so only the gateway crate recompiles on -# a source-only change. python3-dev is present in every rust stage because the -# `python-config` feature links libpython via pyo3 (even in the cook step), and -# python3-pip builds the litellm wheel in the builder stage. -FROM rust:1.98-slim-bookworm AS chef -ENV PYO3_PYTHON=python3.11 -# rustup reads rust-toolchain.toml from any parent of the working directory, so -# copying it in is what keeps every cargo call below on the repo's pinned -# channel rather than on whatever the base image happens to ship. -COPY rust-toolchain.toml /build/rust-toolchain.toml -WORKDIR /build/litellm-rust -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - python3 python3-dev python3-pip pkg-config libssl-dev clang \ - && rm -rf /var/lib/apt/lists/* \ - && cargo install cargo-chef --locked --version 0.1.77 - -# ---- Planner ---------------------------------------------------------------- -# Produce the dependency recipe from the rust workspace manifests + Cargo.lock. -FROM chef AS planner -COPY litellm-rust/ . -RUN cargo chef prepare --recipe-path recipe.json - -# ---- Builder ---------------------------------------------------------------- -FROM chef AS builder -# Cook (compile) just the dependencies first — this layer is cached and reused -# whenever only gateway source changes. -COPY --from=planner /build/litellm-rust/recipe.json recipe.json -RUN cargo chef cook --locked --release \ - -p litellm-ai-gateway --features server,python-config \ - --recipe-path recipe.json -# Now copy the real sources and build the gateway binary. Deps are already cooked -# above, so this step only recompiles the gateway crate. -COPY litellm-rust/ . -RUN cargo build --locked --release -p litellm-ai-gateway --bin litellm-ai-gateway --features server,python-config - -# The root pyproject builds with maturin against litellm-rust/crates/python-bridge, -# so the wheel is built here, next to the crate sources and the cargo toolchain, -# and the runtime stage installs the artifact instead of compiling anything. -# litellm[proxy] pins litellm-enterprise and litellm-proxy-extras to the versions -# in this repo, and those hit PyPI hours after every version bump merges, so both -# wheels are built from the repo too instead of being resolved from PyPI. -COPY pyproject.toml README.md LICENSE /build/ -COPY litellm/ /build/litellm/ -COPY enterprise/ /build/enterprise/ -COPY litellm-proxy-extras/ /build/litellm-proxy-extras/ -RUN pip3 wheel --no-cache-dir --no-deps --wheel-dir /build/dist \ - /build /build/enterprise /build/litellm-proxy-extras - -# ---- Runtime ---------------------------------------------------------------- -# python:3.11-slim-bookworm ships libpython3.11, matching the builder's PyO3 -# 3.11 ABI so the embedded interpreter links and imports cleanly. -FROM python:3.11-slim-bookworm AS runtime - -# CA certificates for outbound TLS to the OpenAI realtime endpoint. -RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app - -# Install litellm (with proxy extras) FROM THIS REPO'S SOURCE so -# `import litellm.proxy.read_model_list` works — it is not on PyPI yet. The two -# sibling wheels come from the builder as well, so the pins in litellm[proxy] -# resolve against them and never wait on a PyPI publish. -COPY --from=builder /build/dist/*.whl /tmp/wheels/ -RUN wheel="$(ls /tmp/wheels/litellm-*.whl)" \ - && pip install --no-cache-dir \ - /tmp/wheels/litellm_enterprise-*.whl \ - /tmp/wheels/litellm_proxy_extras-*.whl \ - "${wheel}[proxy]" \ - && rm -rf /tmp/wheels - -# The compiled gateway binary (pure-Rust realtime hot path; Python is load-time -# only). -COPY --from=builder /build/litellm-rust/target/release/litellm-ai-gateway /usr/local/bin/litellm-ai-gateway - -# Default config.yaml. A real deploy can override this (e.g. mount a Render -# secret file at the same path) — never bake secrets into the image. -COPY litellm-rust/crates/ai-gateway/config.yaml /app/config.yaml - -# Bind to all interfaces (Render routes to 0.0.0.0:$PORT) and load the model_list -# from config.yaml via the embedded python config reader. -ENV HOST=0.0.0.0 \ - LITELLM_CONFIG_PATH=/app/config.yaml - -# Drop to a non-root user. The realtime hot path needs no root privileges, so -# running unprivileged limits blast radius if the process is ever compromised. -# The binary in /usr/local/bin is world-executable (COPY default mode 755); we -# only need /app (and the config.yaml it reads) owned by the unprivileged user. -RUN useradd --system --no-create-home --uid 10001 appuser \ - && chown -R appuser:appuser /app -USER appuser - -ENTRYPOINT ["/usr/local/bin/litellm-ai-gateway"] diff --git a/litellm-rust/crates/ai-gateway/Dockerfile.dockerignore b/litellm-rust/crates/ai-gateway/Dockerfile.dockerignore deleted file mode 100644 index d1386ff684d..00000000000 --- a/litellm-rust/crates/ai-gateway/Dockerfile.dockerignore +++ /dev/null @@ -1,54 +0,0 @@ -# Dockerfile-specific ignore-file for the Rust AI Gateway build. -# -# The build context is the repo root (so the image can pip install litellm from -# source AND build the rust workspace). BuildKit honors `.dockerignore` -# next to the Dockerfile and it takes precedence over the repo-root `.dockerignore`, -# so this file shrinks the (large) repo-root context for THIS build only without -# touching the root `.dockerignore` used by the main litellm images. -# -# Strategy: ignore everything, then re-include only what the build needs: -# - litellm/ (pip install . needs the full package + proxy reader) -# - litellm-rust/ (the rust workspace; Cargo.lock + crate sources) -# - enterprise/ (litellm/proxy/enterprise symlinks into it; maturin walks it) -# - litellm-proxy-extras/ (built into a wheel alongside enterprise/ for litellm[proxy]) -# - pyproject.toml / README.md / LICENSE (packaging metadata for the wheel build) -# - rust-toolchain.toml (the pinned channel every cargo call in the build uses) -* - -# --- re-include the build inputs --- -!litellm/ -!litellm-rust/ -!enterprise/ -!litellm-proxy-extras/ -!pyproject.toml -!rust-toolchain.toml -!README.md -!LICENSE - -# --- prune heavy / irrelevant subpaths back out of the re-included trees --- -# Rust build artifacts (huge; regenerated in the builder). -**/target/ -# Committed python distribution artifacts; the wheel build does not read them. -enterprise/dist/ -litellm-proxy-extras/dist/ -# Python caches and compiled bytecode. -**/__pycache__/ -**/*.pyc -**/*.pyo -**/.pytest_cache/ -**/.ruff_cache/ -**/.mypy_cache/ -# Node / UI build output bundled under the python package (not needed to import -# litellm.proxy.read_model_list). -**/node_modules/ -litellm/proxy/_experimental/out/ -# Tests, logs, and local scratch. -**/tests/ -**/test/ -*.log -log.txt -*.tgz -# VCS / editor / CI metadata that may live under re-included trees. -**/.git/ -.git/ -**/.DS_Store diff --git a/litellm-rust/crates/ai-gateway/README.md b/litellm-rust/crates/ai-gateway/README.md deleted file mode 100644 index cbcd8119546..00000000000 --- a/litellm-rust/crates/ai-gateway/README.md +++ /dev/null @@ -1,206 +0,0 @@ -# LiteLLM Rust AI Gateway - -A minimal Axum service that fronts OpenAI's realtime API. Clients open a -WebSocket to `GET /v1/realtime`; the gateway authenticates, selects a deployment, -dials OpenAI upstream, and splices the two sockets frame-by-frame. - -## Crates - -`litellm-rust` has six crates. A crate is a layer or shared foundation, not a route: - -| Crate | Role | -|-------|------| -| litellm-core | The LiteLLM SDK in Rust — per-route entrypoints (`messages::messages()`) that resolve the provider, transform, and make the call; plus types, provider transforms, and the router. | -| litellm-token-counter | Standalone input token counting shared by host integrations without pulling in the full SDK. | -| litellm-config | Config-loading boundary. Returns resolved deployments and optionally delegates loading to Python. | -| litellm-ai-gateway | The Axum server (behind the `server` feature) and WebSocket hosts. Translates HTTP/WS to core entrypoints; no provider handlers. | -| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. | -| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. | - -Dependency direction is acyclic: config depends on core, the gateway depends on config and core, and the Python bridge depends on the domain layers, token counter, and Python interop. - -- **Client endpoint:** `wss:///v1/realtime?model=` (WebSocket) -- **Auth:** `Authorization: Bearer $LITELLM_MASTER_KEY` (fails closed if unset) -- **Health:** `GET /health/readiness`, `GET /health/liveness` -- **Request logs:** POSTed to a LiteLLM proxy at `/v1/rust_control_plane/logs` (see [Request logging](#request-logging)) - -> **Realtime serving is pure Rust.** Python is used at **load time only** — to -> read the config once at boot. The realtime hot path never touches Python. - -The former `/health/gil` route and its acquisition counter were removed. They -only observed the single startup config load and did not prove that every GIL -acquisition was instrumented - -## Configuration (config.yaml) - -The gateway loads its `model_list` from a **config.yaml**, the same as the -LiteLLM proxy. Point `LITELLM_CONFIG_PATH` at the file: - -```yaml -# config.yaml -model_list: - - model_name: gpt-realtime - litellm_params: - model: openai/gpt-realtime - api_key: os.environ/OPENAI_API_KEY -``` - -```bash -LITELLM_CONFIG_PATH=./config.yaml ./litellm-ai-gateway -``` - -At boot `litellm-config` calls into `litellm.proxy.read_model_list` and returns -resolved deployments to the gateway, which constructs the router. The Python -backend still reuses the **real proxy config reader** (`ProxyConfig.get_config`), -so everything the proxy supports in config.yaml works here too: - -- `include:` to merge in other config files, -- `os.environ/VAR` secret references (resolved via the secret manager, never - inlined), -- DB-stored models (when a database is configured). - -Secrets stay out of the config — reference them with `os.environ/...` and set -the env var at deploy time. The shipped Docker image is built with the -`python-config` feature and **bundles litellm**, so config loading works out of -the box; the default baked config lives at `/app/config.yaml` and can be -overridden at deploy time (e.g. a Render secret file mounted at the same path). - -### Environment variables - -| Var | Required | Default | Purpose | -|---|---|---|---| -| `LITELLM_CONFIG_PATH` | yes (config mode) | — | Path to the config.yaml the gateway loads its `model_list` from. The Docker image defaults this to `/app/config.yaml`. | -| `LITELLM_MASTER_KEY` | yes | — | Bearer token clients must send. Unset ⇒ all `/v1/realtime` requests are rejected (fail closed). | -| `OPENAI_API_KEY` | yes | — | Upstream OpenAI key. Referenced by config.yaml as `os.environ/OPENAI_API_KEY` for the gateway→OpenAI dial. | -| `HOST` | no | `127.0.0.1` | **Set to `0.0.0.0` in any container/deploy** or external traffic is refused. | -| `PORT` | no | `4001` | Listen port. Render and most PaaS inject this automatically. | -| `LITELLM_PROXY_BASE_URL` | no | `http://localhost:4000` | LiteLLM proxy that request logs are POSTed to. See [Request logging](#request-logging). | - -> Secrets (`LITELLM_MASTER_KEY`, `OPENAI_API_KEY`) are never baked into the image -> or `render.yaml` — inject them at deploy time only. - -### Lean env stand-in (fallback) - -If the binary is built **without** `python-config` (default features), or -`LITELLM_CONFIG_PATH` is unset, the gateway falls back to a single-deployment -stand-in built from the environment: - -| Var | Default | Purpose | -|---|---|---| -| `OPENAI_REALTIME_MODEL` | `gpt-realtime` | The single deployment's model name (also the `?model=` clients pass). | - -The default workspace build links no libpython and needs no config file. This -fallback mode only supports one hard-coded OpenAI deployment. **config.yaml is the recommended path** — use the -stand-in only for the leanest possible build. - -## Request logging - -The gateway runs no spend logic. When a session ends it builds one -`StandardLoggingPayload` and POSTs it to `{LITELLM_PROXY_BASE_URL}/v1/rust_control_plane/logs` -(admin-only, bearer = `LITELLM_MASTER_KEY`), and the proxy replays it through its -normal callbacks (spend logs, Langfuse, etc.). The POST is non-blocking: a bounded -channel drained by a background worker, dropping with a counter if the proxy is -down. It sends one payload per session. Both env vars are in the table above. - -Worker tuning, rarely needed: `LITELLM_LOG_CHANNEL_CAPACITY` (4096), -`LITELLM_LOG_BATCH_SIZE` (256), `LITELLM_LOG_FLUSH_INTERVAL_MS` (500). - -## Build & run with Docker - -The image is built `--features server,python-config` and installs litellm **from this -repo's source** (the config reader is newer than any PyPI release), so the build -**context is the repo root**: - -```bash -# from the repo root -docker build -f litellm-rust/crates/ai-gateway/Dockerfile -t litellm-ai-gateway . - -docker run --rm -p 4001:4001 \ - -e HOST=0.0.0.0 -e PORT=4001 \ - -e LITELLM_MASTER_KEY=sk-local \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - litellm-ai-gateway # LITELLM_CONFIG_PATH defaults to /app/config.yaml - -# smoke test -curl -s -o /dev/null -w '%{http_code}\n' localhost:4001/health/readiness # -> 200 -curl -s -o /dev/null -w '%{http_code}\n' localhost:4001/v1/realtime # -> 401 (auth fails closed) -``` - -On boot you should see `loaded model_list from /app/config.yaml via python -config reader` — that confirms the config path (not the env stand-in fallback). -To use your own config, mount it over the default: - -```bash -docker run --rm -p 4001:4001 \ - -e HOST=0.0.0.0 -e LITELLM_MASTER_KEY=sk-local -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -v $(pwd)/my-config.yaml:/app/config.yaml:ro \ - litellm-ai-gateway -``` - -### Cargo-only (no Docker) - -```bash -# config.yaml mode — needs litellm importable in the active python env -LITELLM_CONFIG_PATH=./crates/ai-gateway/config.yaml \ - cargo run --release -p litellm-ai-gateway --features server,python-config - -# env stand-in mode — no python, no config -cargo run --release -p litellm-ai-gateway --features server -``` - -## Deploy on Render - -The service is a Docker **web service**; Render terminates TLS and supports -WebSockets, so the public endpoint is `wss://.onrender.com/v1/realtime`. - -### Option A — Blueprint (`render.yaml`) - -`crates/ai-gateway/render.yaml` describes the service (Docker runtime, -`healthCheckPath: /health/readiness`, repo-root `dockerContext: .`, -`dockerfilePath: ./litellm-rust/crates/ai-gateway/Dockerfile`, -`LITELLM_CONFIG_PATH: /app/config.yaml`). `LITELLM_MASTER_KEY` and -`OPENAI_API_KEY` are `sync: false` — set them in the dashboard after the first -deploy. To use a non-default model_list, mount a **Render Secret File** at -`/app/config.yaml`. Point a Render Blueprint at this repo/branch and apply. - -### Option B — Render API - -```bash -# create a Docker web service from this repo+branch, then set env vars: -curl -X POST https://api.render.com/v1/services \ - -H "Authorization: Bearer $RENDER_API_KEY" -H "Content-Type: application/json" \ - -d '{ - "type": "web_service", "name": "litellm-rust-ai-gateway", - "ownerId": "", "repo": "https://github.com/BerriAI/litellm", - "branch": "", - "serviceDetails": { - "env": "docker", - "envSpecificDetails": { - "dockerfilePath": "./litellm-rust/crates/ai-gateway/Dockerfile", - "dockerContext": "." - }, - "healthCheckPath": "/health/readiness" - } - }' -# then set env vars LITELLM_MASTER_KEY, OPENAI_API_KEY, HOST=0.0.0.0, -# LITELLM_CONFIG_PATH=/app/config.yaml -``` - -Health check path **must** be `/health/readiness`. `autoDeploy` is off by default -in the blueprint — trigger deploys manually (or flip it on) to pick up new commits. - -## Scaling - -Concurrency is what matters, not total connections: each in-flight session holds -one client socket + one upstream socket. To scale, raise the instance count / -enable autoscaling on the Render service (e.g. baseline 10, max 100). Each -instance needs file descriptors for `2 × peak_concurrent_sessions` — raise -`ulimit -n` if you push very high concurrency. - -## Latency note - -The gateway adds the cost of one extra hop: client→gateway, then a fresh -gateway→OpenAI realtime handshake (TLS + WS upgrade + `session.created`). In -benchmarks this is ~100–150 ms of added session-establishment time; first-audio -and steady-state streaming add no measurable overhead. To minimize it, deploy the -gateway in the Render region with the lowest RTT to OpenAI's realtime endpoint. diff --git a/litellm-rust/crates/ai-gateway/config.yaml b/litellm-rust/crates/ai-gateway/config.yaml deleted file mode 100644 index 321801f6862..00000000000 --- a/litellm-rust/crates/ai-gateway/config.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Sample realtime config for the LiteLLM Rust AI Gateway. -# -# litellm-config resolves this model_list at boot through the Python config -# reader (litellm.proxy.read_model_list), then the gateway builds its router. -# Includes, environment secrets, and database-stored models still work. -# -# Secrets are referenced (never inlined) via os.environ/. A real deploy can -# override this file (e.g. mount a Render secret file at LITELLM_CONFIG_PATH). -model_list: - - model_name: gpt-realtime - litellm_params: - model: openai/gpt-realtime - api_key: os.environ/OPENAI_API_KEY diff --git a/litellm-rust/crates/ai-gateway/render.yaml b/litellm-rust/crates/ai-gateway/render.yaml deleted file mode 100644 index 4170849f65d..00000000000 --- a/litellm-rust/crates/ai-gateway/render.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Render blueprint for the LiteLLM Rust AI Gateway (realtime WebSocket proxy). -# -# Single instance for now (no autoscaling). The public endpoint is a -# WebSocket served over TLS: wss://.onrender.com/v1/realtime -# -# Paths are relative to the **repo root** (Render's convention). The build -# context is the repo root so the image can install litellm from source — the -# gateway loads its model_list via litellm.proxy.read_model_list at boot. -# -# Secrets (LITELLM_MASTER_KEY, OPENAI_API_KEY) are marked sync: false — set -# them in the Render dashboard or via the API, never inline here. -services: - - type: web - name: litellm-rust-ai-gateway - runtime: docker - plan: standard - dockerfilePath: ./litellm-rust/crates/ai-gateway/Dockerfile - dockerContext: . - healthCheckPath: /health/readiness - numInstances: 1 - envVars: - # The gateway loads its model_list from this config.yaml via the embedded - # python config reader. The image bakes a default config at /app/config.yaml; - # a real deploy can override it by mounting a Render secret file at this - # same path (Dashboard → Environment → Secret Files) — never inline secrets. - - key: LITELLM_CONFIG_PATH - value: /app/config.yaml - - key: HOST - value: 0.0.0.0 - # Bearer token clients must send on /v1/realtime (fail closed if unset). - - key: LITELLM_MASTER_KEY - sync: false - # Referenced by config.yaml as os.environ/OPENAI_API_KEY for the upstream dial. - - key: OPENAI_API_KEY - sync: false diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs deleted file mode 100644 index b17f17de11f..00000000000 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/hooks.rs +++ /dev/null @@ -1,288 +0,0 @@ -use litellm_core::audio_transcription::{ - AudioTranscriptionRequest as CoreAudioTranscriptionRequest, ProviderAudioTranscriptionRequest, - prepare_audio_transcription_provider_call, -}; -use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleHooks, CallLifecycleTiming}; -use litellm_core::error::Error; -use serde_json::{Map, Value, json}; -use std::future::Future; -use std::pin::Pin; - -use super::types::PreparedAudioTranscriptionRequest; -use crate::integrations::custom_guardrail::{ - CustomGuardrailRunner, GuardrailContext, GuardrailError, GuardrailRequest, -}; -use crate::integrations::custom_logger::{ - CallType, CallbackTiming, CallbackValue, CustomLoggerRunner, LoggingError, ModelCallDetails, -}; -use crate::integrations::types::{ - RequestMetadata, StandardLoggingMetadata, StandardLoggingPayload, -}; - -pub(crate) struct AudioTranscriptionLifecycleHooks { - logger_runner: CustomLoggerRunner, - guardrail_runner: CustomGuardrailRunner, - request_metadata: RequestMetadata, -} - -type AudioFuture<'a, T> = Pin> + Send + 'a>>; -type AudioLogFuture<'a> = Pin + Send + 'a>>; - -impl AudioTranscriptionLifecycleHooks { - pub(crate) fn new( - logger_runner: CustomLoggerRunner, - guardrail_runner: CustomGuardrailRunner, - request_metadata: RequestMetadata, - ) -> Self { - Self { - logger_runner, - guardrail_runner, - request_metadata, - } - } - - async fn run_pre_call_guardrails( - &self, - request: PreparedAudioTranscriptionRequest, - ) -> Result { - if self.guardrail_runner.is_empty() { - return Ok(request); - } - let (guardrail_request, _) = self - .guardrail_runner - .run_pre_call( - &guardrail_context(&self.request_metadata), - GuardrailRequest::new(json!({ - "model": request.model, - "custom_llm_provider": request.custom_llm_provider, - "audio": request.audio, - "optional_params": request.optional_params, - })), - ) - .await - .map_err(guardrail_error_to_core_error)?; - let Value::Object(mut data) = guardrail_request.data else { - return Err(Error::InvalidRequest( - "audio transcription pre_call guardrail must return an object".to_string(), - )); - }; - let audio = data.remove("audio").ok_or_else(|| { - Error::InvalidRequest("audio transcription guardrail removed audio".to_string()) - })?; - let optional_params = match data.remove("optional_params") { - Some(Value::Object(value)) => value, - Some(_) => { - return Err(Error::InvalidRequest( - "audio transcription optional_params must be an object".to_string(), - )); - } - None => Map::new(), - }; - Ok(PreparedAudioTranscriptionRequest { - audio, - optional_params, - ..request - }) - } - - async fn prepare_provider_request( - &self, - request: PreparedAudioTranscriptionRequest, - ) -> Result { - let PreparedAudioTranscriptionRequest { - model, - custom_llm_provider, - audio, - api_key, - api_base, - extra_headers, - optional_params, - timeout, - .. - } = request; - let provider_request = - prepare_audio_transcription_provider_call(CoreAudioTranscriptionRequest { - model: &model, - audio, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: Some(&custom_llm_provider), - extra_headers, - optional_params, - timeout, - })?; - self.run_during_call_guardrails(provider_request).await - } - - async fn run_during_call_guardrails( - &self, - request: ProviderAudioTranscriptionRequest, - ) -> Result { - if self.guardrail_runner.is_empty() { - return Ok(request); - } - let (guardrail_request, _) = self - .guardrail_runner - .run_during_call( - &guardrail_context(&self.request_metadata), - GuardrailRequest::new(json!({ - "model": request.model(), - "custom_llm_provider": request.custom_llm_provider(), - "url": request.url(), - "body": request.body(), - })), - ) - .await - .map_err(guardrail_error_to_core_error)?; - let Value::Object(mut data) = guardrail_request.data else { - return Err(Error::InvalidRequest( - "audio transcription during_call guardrail must return an object".to_string(), - )); - }; - let body = data.remove("body").ok_or_else(|| { - Error::InvalidRequest("audio transcription guardrail removed body".to_string()) - })?; - Ok(request.with_body(body)) - } - - fn logging_payload( - &self, - context: &CallLifecycleContext, - timing: &CallLifecycleTiming, - ) -> StandardLoggingPayload { - StandardLoggingPayload { - id: context.litellm_call_id.clone(), - litellm_call_id: context.litellm_call_id.clone(), - call_type: context.call_type.clone(), - model: context.model.clone(), - custom_llm_provider: context.custom_llm_provider.clone(), - response_cost: 0.0, - prompt_tokens: 0, - completion_tokens: 0, - total_tokens: 0, - start_time: timing.start_time, - end_time: timing.end_time, - stream: false, - metadata: StandardLoggingMetadata { - user_api_key_hash: self.request_metadata.user_api_key_hash.clone(), - user_api_key_user_id: self.request_metadata.user_api_key_user_id.clone(), - user_api_key_team_id: self.request_metadata.user_api_key_team_id.clone(), - ..Default::default() - }, - messages: None, - } - } -} - -impl CallLifecycleHooks - for AudioTranscriptionLifecycleHooks -{ - type PreCallFuture<'a> = AudioFuture<'a, PreparedAudioTranscriptionRequest>; - type DuringCallFuture<'a> = AudioFuture<'a, ProviderAudioTranscriptionRequest>; - type SuccessFuture<'a> = AudioLogFuture<'a>; - type FailureFuture<'a> = AudioLogFuture<'a>; - - fn async_pre_call_hook<'a>( - &'a self, - _context: &'a CallLifecycleContext, - request: PreparedAudioTranscriptionRequest, - ) -> Self::PreCallFuture<'a> { - Box::pin(async move { self.run_pre_call_guardrails(request).await }) - } - - fn async_during_call_hook<'a>( - &'a self, - _context: &'a CallLifecycleContext, - request: PreparedAudioTranscriptionRequest, - ) -> Self::DuringCallFuture<'a> { - Box::pin(async move { self.prepare_provider_request(request).await }) - } - - fn async_log_success_event<'a>( - &'a self, - context: &'a CallLifecycleContext, - response: &'a Value, - timing: &'a CallLifecycleTiming, - ) -> Self::SuccessFuture<'a> { - Box::pin(async move { - if self.logger_runner.is_empty() { - return; - } - self.logger_runner - .async_log_success_event( - &ModelCallDetails::from_standard_logging_payload( - self.logging_payload(context, timing), - ), - &CallbackValue::new("audio_transcription", response.clone()), - CallbackTiming::new(timing.start_time, timing.end_time), - ) - .await; - }) - } - - fn async_log_failure_event<'a>( - &'a self, - context: &'a CallLifecycleContext, - error: &'a Error, - timing: &'a CallLifecycleTiming, - ) -> Self::FailureFuture<'a> { - Box::pin(async move { - if self.logger_runner.is_empty() { - return; - } - let logging_error = LoggingError { - message: error.to_string(), - kind: core_error_kind(error).to_string(), - }; - self.logger_runner - .async_log_failure_event( - &ModelCallDetails::from_standard_logging_payload( - self.logging_payload(context, timing), - ) - .with_failure_error(logging_error.clone()), - Some(&CallbackValue::new( - "error", - json!({"message": logging_error.message, "kind": logging_error.kind}), - )), - CallbackTiming::new(timing.start_time, timing.end_time), - ) - .await; - }) - } -} - -fn guardrail_context(metadata: &RequestMetadata) -> GuardrailContext { - GuardrailContext { - call_type: CallType::Other("audio_transcription".to_string()), - selected_guardrails: Vec::new(), - metadata: std::collections::HashMap::new(), - user_api_key_hash: metadata.user_api_key_hash.clone(), - user_api_key_user_id: metadata.user_api_key_user_id.clone(), - user_api_key_team_id: metadata.user_api_key_team_id.clone(), - trace_parent: None, - } -} - -fn guardrail_error_to_core_error(error: GuardrailError) -> Error { - Error::InvalidRequest(format!("{}: {}", error.kind, error.message)) -} - -fn core_error_kind(error: &Error) -> &'static str { - match error { - Error::Auth(_) - | Error::MissingApiKey { .. } - | Error::MissingAzureAiCredentials - | Error::MissingAzureDocumentIntelligenceCredentials - | Error::MissingReductoApiKey => "AuthError", - Error::InvalidProvider(_) => "InvalidProvider", - Error::InvalidRequest(_) => "InvalidRequest", - Error::InvalidType { .. } => "InvalidType", - Error::MissingField(_) | Error::MissingDocumentUrl => "MissingField", - Error::Http { .. } => "HttpError", - Error::InvalidResponse(_) => "InvalidResponse", - Error::Network(_) => "NetworkError", - Error::Connect(_) => "ConnectError", - Error::Routing(_) => "RoutingError", - Error::Unsupported(_) => "UnsupportedRequest", - } -} diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs deleted file mode 100644 index 03d621b8414..00000000000 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/mod.rs +++ /dev/null @@ -1,23 +0,0 @@ -use litellm_core::Error; -use litellm_core::audio_transcription::execute_audio_transcription_provider_call; -use litellm_core::call_lifecycle::CallLifecycle; -use serde_json::Value; - -mod hooks; -mod prepare; -mod types; - -pub use types::AudioTranscriptionRequest; - -use prepare::{PreparedAudioTranscriptionCall, prepare_audio_transcription_call}; - -pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result { - let PreparedAudioTranscriptionCall { request, hooks } = - prepare_audio_transcription_call(request); - CallLifecycle::default() - .run_request(request, &hooks, execute_audio_transcription_provider_call) - .await -} - -#[cfg(test)] -mod tests; diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/prepare.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/prepare.rs deleted file mode 100644 index a475d58635f..00000000000 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/prepare.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{SystemTime, UNIX_EPOCH}; - -use litellm_core::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; - -use super::hooks::AudioTranscriptionLifecycleHooks; -use super::types::{AudioTranscriptionRequest, PreparedAudioTranscriptionRequest}; -use crate::integrations::custom_guardrail::CustomGuardrailRunner; -use crate::integrations::custom_logger::CustomLoggerRunner; - -pub(crate) struct PreparedAudioTranscriptionCall { - pub(crate) request: PreparedAudioTranscriptionRequest, - pub(crate) hooks: AudioTranscriptionLifecycleHooks, -} - -pub(crate) fn prepare_audio_transcription_call( - request: AudioTranscriptionRequest<'_>, -) -> PreparedAudioTranscriptionCall { - let call_id = request - .litellm_call_id - .map(str::to_string) - .unwrap_or_else(new_audio_transcription_call_id); - let provider_info = get_custom_llm_provider(request.model, request.custom_llm_provider) - .unwrap_or(CustomLlmProvider { - model: request.model, - custom_llm_provider: "bedrock", - }); - PreparedAudioTranscriptionCall { - request: PreparedAudioTranscriptionRequest { - model: provider_info.model.to_string(), - custom_llm_provider: provider_info.custom_llm_provider.to_string(), - litellm_call_id: call_id, - audio: request.audio, - api_key: request.api_key.map(str::to_string), - api_base: request.api_base.map(str::to_string), - extra_headers: request.extra_headers, - optional_params: request.optional_params, - timeout: request.timeout, - }, - hooks: AudioTranscriptionLifecycleHooks::new( - CustomLoggerRunner::new(request.callbacks), - CustomGuardrailRunner::new(request.guardrails), - request.request_metadata, - ), - } -} - -fn new_audio_transcription_call_id() -> String { - static COUNTER: AtomicU64 = AtomicU64::new(1); - let sequence = COUNTER.fetch_add(1, Ordering::Relaxed); - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_or(0, |duration| duration.as_nanos()); - format!("audio-transcription-{timestamp}-{sequence}") -} diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/tests.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/tests.rs deleted file mode 100644 index 5df04708b7d..00000000000 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/tests.rs +++ /dev/null @@ -1,53 +0,0 @@ -use std::io::{Read, Write}; -use std::net::TcpListener; -use std::thread; - -use serde_json::{Map, json}; - -use super::{AudioTranscriptionRequest, audio_transcription}; - -#[tokio::test] -async fn bedrock_request_is_signed_and_contains_audio() { - let listener = TcpListener::bind("127.0.0.1:0").expect("listener"); - let address = listener.local_addr().expect("address"); - let server = thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("connection"); - let mut request = Vec::new(); - let mut buffer = [0_u8; 16_384]; - let count = stream.read(&mut buffer).expect("request"); - request.extend_from_slice(&buffer[..count]); - let request = String::from_utf8_lossy(&request); - assert!(request.contains("POST /model/mistral.voxtral-mini-3b-2507/converse")); - assert!(request.contains("authorization: AWS4-HMAC-SHA256")); - assert!(request.contains("x-amz-date:")); - assert!(request.contains("\"bytes\":\"AQI=\"")); - assert!(request.contains("Transcribe the audio. Respond with only the transcript.")); - let response = b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 53\r\nConnection: close\r\n\r\n{\"output\":{\"message\":{\"content\":[{\"text\":\"hello\"}]}}}"; - stream.write_all(response).expect("response"); - }); - - let optional_params = Map::from_iter([ - ("aws_access_key_id".to_string(), json!("access-key")), - ("aws_secret_access_key".to_string(), json!("secret-key")), - ("aws_region_name".to_string(), json!("us-east-1")), - ]); - let api_base = format!("http://{address}"); - let response = audio_transcription(AudioTranscriptionRequest { - model: "mistral.voxtral-mini-3b-2507", - audio: json!({"data": "AQI=", "format": "wav", "filename": "audio.wav"}), - api_key: None, - api_base: Some(&api_base), - custom_llm_provider: Some("bedrock"), - extra_headers: None, - optional_params, - timeout: None, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, - }) - .await - .expect("transcription"); - assert_eq!(response, json!({"text": "hello"})); - server.join().expect("server"); -} diff --git a/litellm-rust/crates/ai-gateway/src/audio_transcription/types.rs b/litellm-rust/crates/ai-gateway/src/audio_transcription/types.rs deleted file mode 100644 index b470638264e..00000000000 --- a/litellm-rust/crates/ai-gateway/src/audio_transcription/types.rs +++ /dev/null @@ -1,47 +0,0 @@ -use std::sync::Arc; -use std::time::Duration; - -use litellm_core::call_lifecycle::{CallLifecycleContext, CallLifecycleRequest}; -use serde_json::{Map, Value}; - -use crate::integrations::custom_guardrail::CustomGuardrail; -use crate::integrations::custom_logger::CustomLogger; -use crate::integrations::types::RequestMetadata; - -pub struct AudioTranscriptionRequest<'a> { - pub model: &'a str, - pub audio: Value, - pub api_key: Option<&'a str>, - pub api_base: Option<&'a str>, - pub custom_llm_provider: Option<&'a str>, - pub extra_headers: Option>, - pub optional_params: Map, - pub timeout: Option, - pub callbacks: Vec>, - pub guardrails: Vec>, - pub request_metadata: RequestMetadata, - pub litellm_call_id: Option<&'a str>, -} - -pub(crate) struct PreparedAudioTranscriptionRequest { - pub(crate) model: String, - pub(crate) custom_llm_provider: String, - pub(crate) litellm_call_id: String, - pub(crate) audio: Value, - pub(crate) api_key: Option, - pub(crate) api_base: Option, - pub(crate) extra_headers: Option>, - pub(crate) optional_params: Map, - pub(crate) timeout: Option, -} - -impl CallLifecycleRequest for PreparedAudioTranscriptionRequest { - fn lifecycle_context(&self) -> CallLifecycleContext { - CallLifecycleContext::new( - "audio_transcription", - self.model.clone(), - self.custom_llm_provider.clone(), - self.litellm_call_id.clone(), - ) - } -} diff --git a/litellm-rust/crates/ai-gateway/src/auth/mod.rs b/litellm-rust/crates/ai-gateway/src/auth/mod.rs deleted file mode 100644 index b09d8285c3a..00000000000 --- a/litellm-rust/crates/ai-gateway/src/auth/mod.rs +++ /dev/null @@ -1,93 +0,0 @@ -//! Gateway authentication, as an axum **extractor** (the idiomatic pattern — -//! keeps handlers clean and auth testable). -//! -//! For now this is a single **master key**: any caller presenting it as -//! `Authorization: Bearer ` may invoke the gateway. Per-key auth, budgets, -//! and rate limits are delegated to the Python proxy in a later phase. -//! -//! A handler opts in by adding [`RequireMasterKey`] to its arguments; auth then -//! runs during extraction, before the handler body. Routes never re-implement it. - -use axum::extract::FromRequestParts; -use axum::http::StatusCode; -use axum::http::header::AUTHORIZATION; -use axum::http::request::Parts; -use sha2::{Digest, Sha256}; -use subtle::ConstantTimeEq; - -use crate::state::AppState; - -/// SHA-256 hex digest of a token — the exact transform the Python proxy applies -/// (`litellm.proxy.utils.hash_token`). -/// -/// STRICT REQUIREMENT: a raw key (`LITELLM_MASTER_KEY`, a virtual key, …) must -/// **never** leave this gateway in a log payload. Spend logs and every callback -/// integration receive `user_api_key_hash`, so that field must be this hash, not -/// the credential. Hashing here also means the value matches the key's hash in -/// `LiteLLM_SpendLogs.api_key`, so realtime spend joins with the rest of LiteLLM. -pub fn hash_token(token: &str) -> String { - let digest = Sha256::digest(token.as_bytes()); - let mut hex = String::with_capacity(digest.len() * 2); - for byte in digest { - use std::fmt::Write; - let _ = write!(hex, "{byte:02x}"); - } - hex -} - -/// Extractor that requires the configured master key as a bearer token. -/// -/// Rejections: `500` when no master key is configured (permanent -/// misconfiguration, not a transient outage); `401` on a missing/incorrect -/// token. The comparison is constant-time. -pub struct RequireMasterKey; - -#[axum::async_trait] -impl FromRequestParts for RequireMasterKey { - type Rejection = (StatusCode, String); - - async fn from_request_parts( - parts: &mut Parts, - state: &AppState, - ) -> Result { - let Some(expected) = state.master_key.as_deref() else { - return Err(( - StatusCode::INTERNAL_SERVER_ERROR, - "gateway auth not configured (set LITELLM_MASTER_KEY)".to_string(), - )); - }; - let provided = parts - .headers - .get(AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.strip_prefix("Bearer ")) - .map(str::trim); - match provided { - Some(token) if bool::from(token.as_bytes().ct_eq(expected.as_bytes())) => Ok(Self), - _ => Err(( - StatusCode::UNAUTHORIZED, - "missing or invalid bearer token".to_string(), - )), - } - } -} - -#[cfg(test)] -mod tests { - use super::hash_token; - - #[test] - fn hash_token_matches_python_sha256_hexdigest() { - // Must equal hashlib.sha256("sk-1234".encode()).hexdigest() — the value - // the proxy stores in LiteLLM_SpendLogs.api_key. - assert_eq!( - hash_token("sk-1234"), - "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b" - ); - // 64 lowercase hex chars, and never the raw input. - let h = hash_token("sk-secret"); - assert_eq!(h.len(), 64); - assert!(h.chars().all(|c| c.is_ascii_hexdigit())); - assert_ne!(h, "sk-secret"); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/bin/trace_parity_gateway.rs b/litellm-rust/crates/ai-gateway/src/bin/trace_parity_gateway.rs deleted file mode 100644 index e247c650fad..00000000000 --- a/litellm-rust/crates/ai-gateway/src/bin/trace_parity_gateway.rs +++ /dev/null @@ -1,42 +0,0 @@ -use std::io::Read; - -use serde::Deserialize; -use serde_json::Value; - -#[derive(Deserialize)] -struct Input { - path: String, - model_alias: String, - provider_model: String, - api_base: String, - body: Value, -} - -#[tokio::main] -async fn main() { - let mut input = String::new(); - if let Err(error) = std::io::stdin().read_to_string(&mut input) { - fail(error); - } - let input: Input = match serde_json::from_str(&input) { - Ok(input) => input, - Err(error) => fail(error), - }; - let result = litellm_ai_gateway::trace_parity::traced_request( - input.path, - input.model_alias, - input.provider_model, - input.api_base, - input.body, - ) - .await; - match serde_json::to_string(&result) { - Ok(result) => println!("{result}"), - Err(error) => fail(error), - } -} - -fn fail(error: impl std::fmt::Display) -> ! { - eprintln!("{error}"); - std::process::exit(1) -} diff --git a/litellm-rust/crates/ai-gateway/src/client.rs b/litellm-rust/crates/ai-gateway/src/client.rs deleted file mode 100644 index ff2606f0229..00000000000 --- a/litellm-rust/crates/ai-gateway/src/client.rs +++ /dev/null @@ -1,14 +0,0 @@ -use std::sync::OnceLock; -use std::time::Duration; - -const HTTP_CLIENT_TIMEOUT_SECS: u64 = 600; - -pub(crate) fn http_client() -> &'static reqwest::Client { - static CLIENT: OnceLock = OnceLock::new(); - CLIENT.get_or_init(|| { - reqwest::Client::builder() - .timeout(Duration::from_secs(HTTP_CLIENT_TIMEOUT_SECS)) - .build() - .expect("failed to build reqwest client") - }) -} diff --git a/litellm-rust/crates/ai-gateway/src/constants.rs b/litellm-rust/crates/ai-gateway/src/constants.rs deleted file mode 100644 index 78af374bf70..00000000000 --- a/litellm-rust/crates/ai-gateway/src/constants.rs +++ /dev/null @@ -1,42 +0,0 @@ -//! Crate-level constants for the ai-gateway. -//! -//! Per `litellm-rust/CLAUDE.md`, magic numbers and fixed strings live here -//! (the Rust mirror of Python's `litellm/constants.py`), not inline in feature -//! modules. Env-overridable tunables keep their `DEFAULT_*` value here; the env -//! read + fallback happens at the host/config layer. - -/// Default LiteLLM control-plane base URL for request-log egress when -/// `LITELLM_PROXY_BASE_URL` is unset. -pub(crate) const DEFAULT_PROXY_BASE_URL: &str = "http://localhost:4000"; - -/// The logs ingest path appended to the proxy base. Not a tunable; it is the -/// proxy's API contract (the rust-control-plane router on the Python proxy). -pub(crate) const RUST_CONTROL_PLANE_LOGS_PATH: &str = "/v1/rust_control_plane/logs"; - -/// Default bounded channel depth for the log-egress worker. -/// Override: `LITELLM_LOG_CHANNEL_CAPACITY`. -pub(crate) const DEFAULT_CHANNEL_CAPACITY: usize = 4096; - -/// Default max records POSTed per request to the control plane. -/// Override: `LITELLM_LOG_BATCH_SIZE`. -pub(crate) const DEFAULT_MAX_BATCH_SIZE: usize = 256; - -/// Default partial-batch flush cadence, in ms. -/// Override: `LITELLM_LOG_FLUSH_INTERVAL_MS`. -pub(crate) const DEFAULT_FLUSH_INTERVAL_MS: u64 = 500; - -/// Provider attributed to realtime sessions in the logging payload. -#[cfg(feature = "server")] -pub(crate) const DEFAULT_PROVIDER: &str = "openai"; - -pub(crate) const DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS: u64 = 10; -pub(crate) const DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS: u64 = 300; - -/// HTTP path for the non-streaming Anthropic Messages route. -#[cfg(feature = "server")] -pub(crate) const MESSAGES_ROUTE_PATH: &str = "/v1/messages"; - -/// Request headers owned by the gateway and never forwarded upstream. -#[cfg(feature = "server")] -pub(crate) const MESSAGES_HEADERS_NOT_FORWARDED: &[&str] = - &["authorization", "connection", "content-length", "host"]; diff --git a/litellm-rust/crates/ai-gateway/src/integrations/README.md b/litellm-rust/crates/ai-gateway/src/integrations/README.md deleted file mode 100644 index 16a162dac57..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/README.md +++ /dev/null @@ -1,127 +0,0 @@ -# LiteLLM Rust integrations - -This directory contains Rust-native equivalents of LiteLLM integration hooks. -The first supported surfaces are terminal custom loggers and pre/during-call -custom guardrails. - -## File layout - -Every integration is a folder: - -- `mod.rs` contains the implementation, trait, runner, or adapter -- `types.rs` contains the integration-local request, response, error, and future - types - -Do not add new flat integration files such as `custom_logger.rs`. Shared wire -contracts that are used by multiple integrations can stay in -`integrations/types.rs`. - -Call ordering and lifecycle timing live in `litellm-core/src/call_lifecycle`. -Call-type modules, such as OCR, adapt their request and response shapes into -that generic lifecycle runner. - -## CustomLogger - -Implement `CustomLogger` when Rust code needs to observe terminal success or -failure events. Method names intentionally match Python `CustomLogger` names. - -```rust -use litellm_ai_gateway::integrations::custom_logger::{ - CallbackTiming, CallbackValue, CustomLogger, LogFuture, ModelCallDetails, -}; - -struct RecordingLogger; - -impl CustomLogger for RecordingLogger { - fn async_log_success_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - response_obj: &'a CallbackValue, - timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - let model = &model_call_details.model; - let provider = &model_call_details.custom_llm_provider; - let call_type = model_call_details.call_type.to_string(); - let request_id = model_call_details.request_id.as_deref(); - let response_object = &response_obj.object; - let duration = timing.end_time - timing.start_time; - let standard_payload = model_call_details.standard_logging_payload.as_ref(); - - Ok(()) - }) - } - - fn async_log_failure_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - response_obj: Option<&'a CallbackValue>, - timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - let error = model_call_details.failure_error.as_ref(); - let response_object = response_obj.map(|value| value.object.as_str()); - let duration = timing.end_time - timing.start_time; - - Ok(()) - }) - } -} -``` - -Use `CustomLoggerRunner` to fan out terminal events to configured loggers. The -runner is a no-op when no loggers are configured, which is the expected fast -path for requests without callbacks. - -## CustomGuardrail - -Implement `CustomGuardrail` when Rust code needs to run pre-call or native -during-call checks. Method names intentionally match Python `CustomGuardrail` -entrypoints inherited from Python `CustomLogger`. - -```rust -use litellm_ai_gateway::integrations::custom_guardrail::{ - CustomGuardrail, GuardrailContext, GuardrailDecision, GuardrailEventHook, - GuardrailFuture, GuardrailRequest, -}; - -struct BlocklistedPromptGuardrail; - -impl CustomGuardrail for BlocklistedPromptGuardrail { - fn guardrail_name(&self) -> &str { - "blocklisted-prompt" - } - - fn supported_event_hooks(&self) -> &[GuardrailEventHook] { - &[GuardrailEventHook::PreCall] - } - - fn async_pre_call_hook<'a>( - &'a self, - _context: &'a GuardrailContext, - request: GuardrailRequest, - ) -> GuardrailFuture<'a> { - Box::pin(async move { - if request.data.to_string().contains("blocked phrase") { - return Ok(GuardrailDecision::Block( - litellm_ai_gateway::integrations::custom_guardrail::GuardrailError::blocked( - "blocked phrase detected", - ), - )); - } - Ok(GuardrailDecision::Allow(request)) - }) - } -} -``` - -Use `CustomGuardrailRunner::run_pre_call` for `pre_call` guardrails and -`CustomGuardrailRunner::run_during_call` for `during_call` guardrails. A -`GuardrailDecision::Mask` continues with modified request data. -`GuardrailDecision::Block` short-circuits the provider call. - -## Current boundary - -These are Rust-only primitives. Python callback and guardrail adapters are a -separate layer that should implement these Rust traits instead of changing the -runner interfaces. diff --git a/litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/mod.rs b/litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/mod.rs deleted file mode 100644 index e5d4ce3a708..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/mod.rs +++ /dev/null @@ -1,468 +0,0 @@ -//! Rust mirror of Python `CustomGuardrail` entrypoints used by the proxy. -//! -//! This module is intentionally Rust-only: Python/PyO3 adapters are a later -//! layer that should implement this trait rather than changing the runner. - -use std::future::Future; -use std::sync::Arc; - -use crate::integrations::custom_logger::{ - CallbackTiming, CallbackValue, CustomLoggerRunner, LoggingError, ModelCallDetails, -}; - -pub mod types; - -pub use types::{ - GuardrailContext, GuardrailDecision, GuardrailDispatchReport, GuardrailError, - GuardrailEventHook, GuardrailFuture, GuardrailRequest, -}; - -pub trait CustomGuardrail: Send + Sync { - fn guardrail_name(&self) -> &str; - - fn supported_event_hooks(&self) -> &[GuardrailEventHook]; - - /// Python 1:1 name: `async_pre_call_hook(user_api_key_dict, cache, data, call_type)`. - fn async_pre_call_hook<'a>( - &'a self, - _context: &'a GuardrailContext, - request: GuardrailRequest, - ) -> GuardrailFuture<'a> { - Box::pin(async move { Ok(GuardrailDecision::Allow(request)) }) - } - - /// Python 1:1 name: `async_moderation_hook(data, user_api_key_dict, call_type)`. - fn async_moderation_hook<'a>( - &'a self, - _context: &'a GuardrailContext, - request: GuardrailRequest, - ) -> GuardrailFuture<'a> { - Box::pin(async move { Ok(GuardrailDecision::Allow(request)) }) - } -} - -pub struct CustomGuardrailRunner { - guardrails: Vec>, -} - -impl CustomGuardrailRunner { - pub fn new(guardrails: Vec>) -> Self { - Self { guardrails } - } - - pub fn is_empty(&self) -> bool { - self.guardrails.is_empty() - } - - pub async fn run_pre_call( - &self, - context: &GuardrailContext, - request: GuardrailRequest, - ) -> Result<(GuardrailRequest, GuardrailDispatchReport), GuardrailError> { - self.run_hook(GuardrailEventHook::PreCall, context, request) - .await - } - - pub async fn run_during_call( - &self, - context: &GuardrailContext, - request: GuardrailRequest, - ) -> Result<(GuardrailRequest, GuardrailDispatchReport), GuardrailError> { - self.run_hook(GuardrailEventHook::DuringCall, context, request) - .await - } - - pub async fn run_before_provider( - &self, - event_hook: GuardrailEventHook, - context: &GuardrailContext, - request: GuardrailRequest, - provider: F, - ) -> Result - where - F: FnOnce(GuardrailRequest) -> Fut, - Fut: Future>, - { - let (request, _) = self.run_hook(event_hook, context, request).await?; - provider(request).await - } - - pub async fn run_pre_call_with_failure_logging( - &self, - context: &GuardrailContext, - request: GuardrailRequest, - logger_runner: &CustomLoggerRunner, - model_call_details: &ModelCallDetails, - timing: CallbackTiming, - ) -> Result<(GuardrailRequest, GuardrailDispatchReport), GuardrailError> { - match self.run_pre_call(context, request).await { - Ok(result) => Ok(result), - Err(error) => { - let failure_details = model_call_details.clone().with_failure_error(LoggingError { - message: error.message.clone(), - kind: error.kind.clone(), - }); - let response_obj = CallbackValue::new( - "guardrail_error", - serde_json::json!({ - "message": error.message, - "kind": error.kind, - }), - ); - logger_runner - .async_log_failure_event(&failure_details, Some(&response_obj), timing) - .await; - Err(error) - } - } - } - - async fn run_hook( - &self, - event_hook: GuardrailEventHook, - context: &GuardrailContext, - mut request: GuardrailRequest, - ) -> Result<(GuardrailRequest, GuardrailDispatchReport), GuardrailError> { - if self.guardrails.is_empty() { - return Ok((request, GuardrailDispatchReport::default())); - } - - let mut report = GuardrailDispatchReport::default(); - for guardrail in &self.guardrails { - if !self.should_run(guardrail.as_ref(), event_hook, context) { - continue; - } - - report.invoked += 1; - let decision = match event_hook { - GuardrailEventHook::PreCall => { - guardrail - .async_pre_call_hook(context, request.clone()) - .await? - } - GuardrailEventHook::DuringCall => { - guardrail - .async_moderation_hook(context, request.clone()) - .await? - } - }; - match decision.into_request() { - Ok(next_request) => request = next_request, - Err(error) => return Err(error), - } - } - - Ok((request, report)) - } - - fn should_run( - &self, - guardrail: &dyn CustomGuardrail, - event_hook: GuardrailEventHook, - context: &GuardrailContext, - ) -> bool { - let supports_hook = guardrail.supported_event_hooks().contains(&event_hook); - let selected = context.selected_guardrails.is_empty() - || context - .selected_guardrails - .iter() - .any(|name| name == guardrail.guardrail_name()); - supports_hook && selected - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::integrations::custom_logger::{CallType, CallbackValue, CustomLogger, LogFuture}; - use crate::integrations::types::{StandardLoggingMetadata, StandardLoggingPayload}; - use serde_json::json; - use std::sync::Mutex; - - #[derive(Clone)] - enum TestDecision { - Allow, - Mask, - Block, - } - - struct RecordingCustomGuardrail { - name: String, - hooks: Vec, - decision: TestDecision, - calls: Mutex>, - } - - impl RecordingCustomGuardrail { - fn new(name: &str, hooks: Vec, decision: TestDecision) -> Self { - Self { - name: name.to_string(), - hooks, - decision, - calls: Mutex::new(Vec::new()), - } - } - - fn calls(&self) -> Vec<&'static str> { - self.calls.lock().unwrap().clone() - } - - fn decision(&self, mut request: GuardrailRequest) -> GuardrailDecision { - match self.decision { - TestDecision::Allow => GuardrailDecision::Allow(request), - TestDecision::Mask => { - request.data["masked"] = json!(true); - GuardrailDecision::Mask(request) - } - TestDecision::Block => { - GuardrailDecision::Block(GuardrailError::blocked("blocked by guardrail")) - } - } - } - } - - impl CustomGuardrail for RecordingCustomGuardrail { - fn guardrail_name(&self) -> &str { - &self.name - } - - fn supported_event_hooks(&self) -> &[GuardrailEventHook] { - &self.hooks - } - - fn async_pre_call_hook<'a>( - &'a self, - _context: &'a GuardrailContext, - request: GuardrailRequest, - ) -> GuardrailFuture<'a> { - Box::pin(async move { - self.calls.lock().unwrap().push("async_pre_call_hook"); - Ok(self.decision(request)) - }) - } - - fn async_moderation_hook<'a>( - &'a self, - _context: &'a GuardrailContext, - request: GuardrailRequest, - ) -> GuardrailFuture<'a> { - Box::pin(async move { - self.calls.lock().unwrap().push("async_moderation_hook"); - Ok(self.decision(request)) - }) - } - } - - #[tokio::test] - async fn pre_call_dispatches_to_async_pre_call_hook() { - let guardrail = Arc::new(RecordingCustomGuardrail::new( - "pre", - vec![GuardrailEventHook::PreCall], - TestDecision::Allow, - )); - let runner = CustomGuardrailRunner::new(vec![guardrail.clone()]); - let context = - GuardrailContext::new(CallType::Ocr).with_selected_guardrails(vec!["pre".to_string()]); - let request = GuardrailRequest::new(json!({"messages": ["hello"]})); - - let (result, report) = runner - .run_pre_call(&context, request) - .await - .expect("guardrail allows request"); - - assert_eq!(report.invoked, 1); - assert_eq!(result.data["messages"], json!(["hello"])); - assert_eq!(guardrail.calls(), vec!["async_pre_call_hook"]); - } - - #[tokio::test] - async fn during_call_dispatches_to_async_moderation_hook() { - let guardrail = Arc::new(RecordingCustomGuardrail::new( - "during", - vec![GuardrailEventHook::DuringCall], - TestDecision::Allow, - )); - let runner = CustomGuardrailRunner::new(vec![guardrail.clone()]); - let context = GuardrailContext::new(CallType::Completion) - .with_selected_guardrails(vec!["during".to_string()]); - let request = GuardrailRequest::new(json!({"prompt": "hello"})); - - let (_result, report) = runner - .run_during_call(&context, request) - .await - .expect("guardrail allows request"); - - assert_eq!(report.invoked, 1); - assert_eq!(guardrail.calls(), vec!["async_moderation_hook"]); - } - - #[tokio::test] - async fn mask_decision_continues_with_updated_request() { - let guardrail = Arc::new(RecordingCustomGuardrail::new( - "masker", - vec![GuardrailEventHook::PreCall], - TestDecision::Mask, - )); - let runner = CustomGuardrailRunner::new(vec![guardrail]); - let context = GuardrailContext::new(CallType::Ocr); - let request = GuardrailRequest::new(json!({"document": "secret"})); - - let (result, report) = runner - .run_pre_call(&context, request) - .await - .expect("mask continues"); - - assert_eq!(report.invoked, 1); - assert_eq!(result.data["masked"], json!(true)); - } - - #[tokio::test] - async fn block_decision_short_circuits_and_logs_failure() { - struct RecordingFailureLogger { - errors: Mutex>, - } - - impl CustomLogger for RecordingFailureLogger { - fn async_log_failure_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - _response_obj: Option<&'a CallbackValue>, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - self.errors.lock().unwrap().push( - model_call_details - .failure_error - .as_ref() - .map(|error| error.kind.clone()) - .unwrap_or_default(), - ); - Ok(()) - }) - } - } - - let guardrail = Arc::new(RecordingCustomGuardrail::new( - "blocker", - vec![GuardrailEventHook::PreCall], - TestDecision::Block, - )); - let guardrail_runner = CustomGuardrailRunner::new(vec![guardrail]); - let logger = Arc::new(RecordingFailureLogger { - errors: Mutex::new(Vec::new()), - }); - let logger_runner = CustomLoggerRunner::new(vec![logger.clone()]); - let context = GuardrailContext::new(CallType::Ocr); - let details = ModelCallDetails::from_standard_logging_payload(StandardLoggingPayload { - id: "req_ocr".to_string(), - litellm_call_id: "req_ocr".to_string(), - call_type: "ocr".to_string(), - model: "mistral-ocr-latest".to_string(), - custom_llm_provider: "mistral".to_string(), - response_cost: 0.0, - prompt_tokens: 0, - completion_tokens: 0, - total_tokens: 0, - start_time: 1.0, - end_time: 1.0, - stream: false, - metadata: StandardLoggingMetadata::default(), - messages: None, - }); - - let err = guardrail_runner - .run_pre_call_with_failure_logging( - &context, - GuardrailRequest::new(json!({"document": "bad"})), - &logger_runner, - &details, - CallbackTiming::new(1.0, 2.0), - ) - .await - .expect_err("guardrail blocks request"); - - assert_eq!(err.kind, "GuardrailBlocked"); - assert_eq!( - logger.errors.lock().unwrap().as_slice(), - ["GuardrailBlocked"] - ); - } - - #[tokio::test] - async fn block_decision_short_circuits_later_guardrails_and_provider_work() { - let blocking_guardrail = Arc::new(RecordingCustomGuardrail::new( - "blocker", - vec![GuardrailEventHook::PreCall], - TestDecision::Block, - )); - let later_guardrail = Arc::new(RecordingCustomGuardrail::new( - "later", - vec![GuardrailEventHook::PreCall], - TestDecision::Allow, - )); - let runner = - CustomGuardrailRunner::new(vec![blocking_guardrail.clone(), later_guardrail.clone()]); - let provider_called = Arc::new(Mutex::new(false)); - let provider_called_for_closure = provider_called.clone(); - - let result = runner - .run_before_provider( - GuardrailEventHook::PreCall, - &GuardrailContext::new(CallType::Completion), - GuardrailRequest::new(json!({"prompt": "blocked"})), - move |_request| async move { - *provider_called_for_closure.lock().unwrap() = true; - Ok("provider response") - }, - ) - .await; - - assert!(result.is_err()); - assert_eq!(blocking_guardrail.calls(), vec!["async_pre_call_hook"]); - assert_eq!(later_guardrail.calls(), Vec::<&'static str>::new()); - assert!(!*provider_called.lock().unwrap()); - } - - #[tokio::test] - async fn run_before_provider_returns_provider_guardrail_error_directly() { - let guardrail = Arc::new(RecordingCustomGuardrail::new( - "allow", - vec![GuardrailEventHook::PreCall], - TestDecision::Allow, - )); - let runner = CustomGuardrailRunner::new(vec![guardrail]); - - let result = runner - .run_before_provider( - GuardrailEventHook::PreCall, - &GuardrailContext::new(CallType::Completion), - GuardrailRequest::new(json!({"prompt": "allowed"})), - |_request| async move { - Err::<&'static str, GuardrailError>(GuardrailError::blocked( - "provider-side guardrail error", - )) - }, - ) - .await; - - let err = result.expect_err("provider error is returned directly"); - assert_eq!(err.kind, "GuardrailBlocked"); - assert_eq!(err.message, "provider-side guardrail error"); - } - - #[tokio::test] - async fn no_guardrails_fast_path_dispatches_nothing() { - let runner = CustomGuardrailRunner::new(Vec::new()); - let context = GuardrailContext::new(CallType::Ocr); - let request = GuardrailRequest::new(json!({"document": "ok"})); - - let (result, report) = runner - .run_pre_call(&context, request) - .await - .expect("no guardrails allow request"); - - assert!(runner.is_empty()); - assert_eq!(report, GuardrailDispatchReport::default()); - assert_eq!(result.data["document"], json!("ok")); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/types.rs b/litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/types.rs deleted file mode 100644 index 825e56cc0d7..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/custom_guardrail/types.rs +++ /dev/null @@ -1,110 +0,0 @@ -use std::collections::HashMap; -use std::future::Future; -use std::pin::Pin; - -use serde_json::Value; - -use crate::integrations::custom_logger::CallType; - -pub type GuardrailFuture<'a> = - Pin> + Send + 'a>>; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum GuardrailEventHook { - PreCall, - DuringCall, -} - -impl GuardrailEventHook { - pub fn as_str(&self) -> &'static str { - match self { - Self::PreCall => "pre_call", - Self::DuringCall => "during_call", - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct GuardrailError { - pub message: String, - pub kind: String, -} - -impl GuardrailError { - pub fn blocked(message: impl Into) -> Self { - Self { - message: message.into(), - kind: "GuardrailBlocked".to_string(), - } - } -} - -impl std::fmt::Display for GuardrailError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}: {}", self.kind, self.message) - } -} - -impl std::error::Error for GuardrailError {} - -#[derive(Clone, Debug)] -pub struct GuardrailContext { - pub call_type: CallType, - pub selected_guardrails: Vec, - pub metadata: HashMap, - pub user_api_key_hash: Option, - pub user_api_key_user_id: Option, - pub user_api_key_team_id: Option, - pub trace_parent: Option, -} - -impl GuardrailContext { - pub fn new(call_type: CallType) -> Self { - Self { - call_type, - selected_guardrails: Vec::new(), - metadata: HashMap::new(), - user_api_key_hash: None, - user_api_key_user_id: None, - user_api_key_team_id: None, - trace_parent: None, - } - } - - pub fn with_selected_guardrails(mut self, selected_guardrails: Vec) -> Self { - self.selected_guardrails = selected_guardrails; - self - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct GuardrailRequest { - pub data: Value, -} - -impl GuardrailRequest { - pub fn new(data: Value) -> Self { - Self { data } - } -} - -#[derive(Clone, Debug, PartialEq)] -pub enum GuardrailDecision { - Allow(GuardrailRequest), - Mask(GuardrailRequest), - Block(GuardrailError), -} - -impl GuardrailDecision { - pub(super) fn into_request(self) -> Result { - match self { - Self::Allow(request) | Self::Mask(request) => Ok(request), - Self::Block(error) => Err(error), - } - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct GuardrailDispatchReport { - pub invoked: usize, -} diff --git a/litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs b/litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs deleted file mode 100644 index 792717dacfc..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs +++ /dev/null @@ -1,317 +0,0 @@ -//! The `CustomLogger` trait — the Rust mirror of Python -//! `litellm/integrations/custom_logger.py::CustomLogger`. -//! -//! The Python-named async terminal methods are the public Rust callback shape. - -use std::sync::Arc; - -pub mod types; - -pub use types::{ - CallType, CallbackDispatchReport, CallbackTiming, CallbackValue, LogError, LogFuture, - LoggingError, ModelCallDetails, -}; - -pub trait CustomLogger: Send + Sync { - /// Python 1:1 name: `async_log_success_event(model_call_details, response_obj, start_time, end_time)`. - fn async_log_success_event<'a>( - &'a self, - _model_call_details: &'a ModelCallDetails, - _response_obj: &'a CallbackValue, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async { Ok(()) }) - } - - /// Python 1:1 name: `async_log_failure_event(model_call_details, response_obj, start_time, end_time)`. - fn async_log_failure_event<'a>( - &'a self, - _model_call_details: &'a ModelCallDetails, - _response_obj: Option<&'a CallbackValue>, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async { Ok(()) }) - } -} - -pub struct CustomLoggerRunner { - loggers: Vec>, -} - -impl CustomLoggerRunner { - pub fn new(loggers: Vec>) -> Self { - Self { loggers } - } - - pub fn is_empty(&self) -> bool { - self.loggers.is_empty() - } - - pub async fn async_log_success_event( - &self, - model_call_details: &ModelCallDetails, - response_obj: &CallbackValue, - timing: CallbackTiming, - ) -> CallbackDispatchReport { - if self.loggers.is_empty() { - return CallbackDispatchReport::default(); - } - - let mut report = CallbackDispatchReport::default(); - for logger in &self.loggers { - report.invoked += 1; - if let Err(err) = logger - .async_log_success_event(model_call_details, response_obj, timing) - .await - { - report.dropped += 1; - eprintln!("litellm-ai-gateway: async_log_success_event dropped: {err}"); - } - } - report - } - - pub async fn async_log_failure_event( - &self, - model_call_details: &ModelCallDetails, - response_obj: Option<&CallbackValue>, - timing: CallbackTiming, - ) -> CallbackDispatchReport { - if self.loggers.is_empty() { - return CallbackDispatchReport::default(); - } - - let mut report = CallbackDispatchReport::default(); - for logger in &self.loggers { - report.invoked += 1; - if let Err(err) = logger - .async_log_failure_event(model_call_details, response_obj, timing) - .await - { - report.dropped += 1; - eprintln!("litellm-ai-gateway: async_log_failure_event dropped: {err}"); - } - } - report - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::integrations::types::{StandardLoggingMetadata, StandardLoggingPayload}; - use serde_json::json; - use std::sync::Mutex; - - #[derive(Clone, Debug, PartialEq)] - struct RecordedEvent { - hook: &'static str, - model: String, - provider: String, - call_type: String, - request_id: Option, - litellm_call_id: Option, - user_id: Option, - response_object: Option, - error_kind: Option, - start_time: f64, - end_time: f64, - standard_logging_model: Option, - } - - #[derive(Default)] - struct RecordingCustomLogger { - events: Mutex>, - } - - impl RecordingCustomLogger { - fn events(&self) -> Vec { - self.events.lock().unwrap().clone() - } - } - - impl CustomLogger for RecordingCustomLogger { - fn async_log_success_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - response_obj: &'a CallbackValue, - timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - self.events.lock().unwrap().push(RecordedEvent { - hook: "async_log_success_event", - model: model_call_details.model.clone(), - provider: model_call_details.custom_llm_provider.clone(), - call_type: model_call_details.call_type.to_string(), - request_id: model_call_details.request_id.clone(), - litellm_call_id: model_call_details.litellm_call_id.clone(), - user_id: model_call_details.metadata.user_api_key_user_id.clone(), - response_object: Some(response_obj.object.clone()), - error_kind: None, - start_time: timing.start_time, - end_time: timing.end_time, - standard_logging_model: model_call_details - .standard_logging_payload - .as_ref() - .map(|payload| payload.model.clone()), - }); - Ok(()) - }) - } - - fn async_log_failure_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - response_obj: Option<&'a CallbackValue>, - timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - self.events.lock().unwrap().push(RecordedEvent { - hook: "async_log_failure_event", - model: model_call_details.model.clone(), - provider: model_call_details.custom_llm_provider.clone(), - call_type: model_call_details.call_type.to_string(), - request_id: model_call_details.request_id.clone(), - litellm_call_id: model_call_details.litellm_call_id.clone(), - user_id: model_call_details.metadata.user_api_key_user_id.clone(), - response_object: response_obj.map(|value| value.object.clone()), - error_kind: model_call_details - .failure_error - .as_ref() - .map(|error| error.kind.clone()), - start_time: timing.start_time, - end_time: timing.end_time, - standard_logging_model: model_call_details - .standard_logging_payload - .as_ref() - .map(|payload| payload.model.clone()), - }); - Ok(()) - }) - } - } - - fn payload(call_type: &str, model: &str, provider: &str) -> StandardLoggingPayload { - StandardLoggingPayload { - id: format!("req_{call_type}"), - litellm_call_id: format!("call_{call_type}"), - call_type: call_type.to_string(), - model: model.to_string(), - custom_llm_provider: provider.to_string(), - response_cost: 0.25, - prompt_tokens: 3, - completion_tokens: 4, - total_tokens: 7, - start_time: 10.0, - end_time: 11.5, - stream: false, - metadata: StandardLoggingMetadata { - user_api_key_hash: Some("hash".to_string()), - user_api_key_user_id: Some("user".to_string()), - user_api_key_team_id: Some("team".to_string()), - ..Default::default() - }, - messages: Some(json!([{"role": "user", "content": "read this"}])), - } - } - - #[tokio::test] - async fn rust_custom_logger_reads_success_payload_for_ocr() { - let logger = Arc::new(RecordingCustomLogger::default()); - let runner = CustomLoggerRunner::new(vec![logger.clone()]); - let details = ModelCallDetails::from_standard_logging_payload(payload( - "ocr", - "mistral-ocr-latest", - "mistral", - )); - let response = CallbackValue::new("ocr", json!({"pages": [{"markdown": "ok"}]})); - let report = runner - .async_log_success_event(&details, &response, CallbackTiming::new(10.0, 11.5)) - .await; - - assert_eq!(report.invoked, 1); - assert_eq!(report.dropped, 0); - assert_eq!( - logger.events(), - vec![RecordedEvent { - hook: "async_log_success_event", - model: "mistral-ocr-latest".to_string(), - provider: "mistral".to_string(), - call_type: "ocr".to_string(), - request_id: Some("req_ocr".to_string()), - litellm_call_id: Some("call_ocr".to_string()), - user_id: Some("user".to_string()), - response_object: Some("ocr".to_string()), - error_kind: None, - start_time: 10.0, - end_time: 11.5, - standard_logging_model: Some("mistral-ocr-latest".to_string()), - }] - ); - } - - #[tokio::test] - async fn rust_custom_logger_reads_failure_payload_for_non_ocr_call_type() { - let logger = Arc::new(RecordingCustomLogger::default()); - let runner = CustomLoggerRunner::new(vec![logger.clone()]); - let details = ModelCallDetails::from_standard_logging_payload(payload( - "acompletion", - "gpt-4.1-mini", - "openai", - )) - .with_failure_error(LoggingError { - message: "provider failed".to_string(), - kind: "ProviderError".to_string(), - }); - let response = CallbackValue::new("error", json!({"message": "provider failed"})); - let report = runner - .async_log_failure_event(&details, Some(&response), CallbackTiming::new(2.0, 3.0)) - .await; - - assert_eq!(report.invoked, 1); - assert_eq!(report.dropped, 0); - assert_eq!( - logger.events(), - vec![RecordedEvent { - hook: "async_log_failure_event", - model: "gpt-4.1-mini".to_string(), - provider: "openai".to_string(), - call_type: "acompletion".to_string(), - request_id: Some("req_acompletion".to_string()), - litellm_call_id: Some("call_acompletion".to_string()), - user_id: Some("user".to_string()), - response_object: Some("error".to_string()), - error_kind: Some("ProviderError".to_string()), - start_time: 2.0, - end_time: 3.0, - standard_logging_model: Some("gpt-4.1-mini".to_string()), - }] - ); - } - - #[tokio::test] - async fn no_callback_fast_path_dispatches_nothing() { - let runner = CustomLoggerRunner::new(Vec::new()); - let details = ModelCallDetails::new("mistral-ocr-latest", "mistral", CallType::Ocr); - let response = CallbackValue::new("ocr", json!({})); - - let report = runner - .async_log_success_event(&details, &response, CallbackTiming::new(1.0, 1.5)) - .await; - - assert!(runner.is_empty()); - assert_eq!(report, CallbackDispatchReport::default()); - } - - #[test] - fn with_standard_logging_payload_keeps_top_level_fields_in_sync() { - let details = ModelCallDetails::new("old-model", "old-provider", CallType::Completion) - .with_standard_logging_payload(payload("ocr", "mistral-ocr-latest", "mistral")); - - assert_eq!(details.model, "mistral-ocr-latest"); - assert_eq!(details.custom_llm_provider, "mistral"); - assert_eq!(details.call_type, CallType::Ocr); - assert_eq!(details.request_id, Some("req_ocr".to_string())); - assert_eq!(details.litellm_call_id, Some("call_ocr".to_string())); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/integrations/custom_logger/types.rs b/litellm-rust/crates/ai-gateway/src/integrations/custom_logger/types.rs deleted file mode 100644 index ba7d67bd46e..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/custom_logger/types.rs +++ /dev/null @@ -1,194 +0,0 @@ -use std::collections::HashMap; -use std::future::Future; -use std::pin::Pin; - -use serde_json::Value; - -use crate::integrations::types::{StandardLoggingMetadata, StandardLoggingPayload}; - -pub type LogFuture<'a> = Pin> + Send + 'a>>; - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct CallbackDispatchReport { - pub invoked: usize, - pub dropped: usize, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum CallType { - Ocr, - Realtime, - Completion, - Acompletion, - ChatCompletion, - Other(String), -} - -impl CallType { - pub fn as_str(&self) -> &str { - match self { - Self::Ocr => "ocr", - Self::Realtime => "realtime", - Self::Completion => "completion", - Self::Acompletion => "acompletion", - Self::ChatCompletion => "chat_completion", - Self::Other(value) => value.as_str(), - } - } -} - -impl From<&str> for CallType { - fn from(value: &str) -> Self { - match value { - "ocr" => Self::Ocr, - "realtime" => Self::Realtime, - "completion" => Self::Completion, - "acompletion" => Self::Acompletion, - "chat_completion" => Self::ChatCompletion, - other => Self::Other(other.to_string()), - } - } -} - -impl std::fmt::Display for CallType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct CallbackTiming { - pub start_time: f64, - pub end_time: f64, -} - -impl CallbackTiming { - pub fn new(start_time: f64, end_time: f64) -> Self { - Self { - start_time, - end_time, - } - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct CallbackValue { - pub object: String, - pub value: Value, -} - -impl CallbackValue { - pub fn new(object: impl Into, value: Value) -> Self { - Self { - object: object.into(), - value, - } - } -} - -#[derive(Clone, Debug)] -pub struct ModelCallDetails { - pub model: String, - pub custom_llm_provider: String, - pub call_type: CallType, - pub metadata: StandardLoggingMetadata, - pub extra_metadata: HashMap, - pub request_id: Option, - pub litellm_call_id: Option, - pub response_cost: Option, - pub standard_logging_payload: Option, - pub failure_error: Option, -} - -impl ModelCallDetails { - pub fn new( - model: impl Into, - custom_llm_provider: impl Into, - call_type: CallType, - ) -> Self { - Self { - model: model.into(), - custom_llm_provider: custom_llm_provider.into(), - call_type, - metadata: StandardLoggingMetadata::default(), - extra_metadata: HashMap::new(), - request_id: None, - litellm_call_id: None, - response_cost: None, - standard_logging_payload: None, - failure_error: None, - } - } - - pub fn from_standard_logging_payload(payload: StandardLoggingPayload) -> Self { - let request_id = Some(payload.id.clone()); - let litellm_call_id = Some(payload.litellm_call_id.clone()); - let response_cost = Some(payload.response_cost); - let metadata = payload.metadata.clone(); - Self { - model: payload.model.clone(), - custom_llm_provider: payload.custom_llm_provider.clone(), - call_type: CallType::from(payload.call_type.as_str()), - metadata, - extra_metadata: HashMap::new(), - request_id, - litellm_call_id, - response_cost, - standard_logging_payload: Some(payload), - failure_error: None, - } - } - - pub fn with_standard_logging_payload(mut self, payload: StandardLoggingPayload) -> Self { - self.model = payload.model.clone(); - self.custom_llm_provider = payload.custom_llm_provider.clone(); - self.call_type = CallType::from(payload.call_type.as_str()); - self.request_id = Some(payload.id.clone()); - self.litellm_call_id = Some(payload.litellm_call_id.clone()); - self.response_cost = Some(payload.response_cost); - self.metadata = payload.metadata.clone(); - self.standard_logging_payload = Some(payload); - self - } - - pub fn with_failure_error(mut self, error: LoggingError) -> Self { - self.failure_error = Some(error); - self - } -} - -#[derive(Clone, Debug)] -pub struct LoggingError { - pub message: String, - pub kind: String, -} - -#[derive(Clone, Debug)] -pub struct LogError { - pub message: String, - pub kind: String, -} - -impl LogError { - pub fn channel_full() -> Self { - Self { - message: "logging channel is full; dropping record".to_string(), - kind: "ChannelFull".to_string(), - } - } - - pub fn channel_closed() -> Self { - Self { - message: "logging channel is closed; worker has shut down".to_string(), - kind: "ChannelClosed".to_string(), - } - } -} - -impl std::fmt::Display for LogError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}: {}", self.kind, self.message) - } -} - -impl std::error::Error for LogError {} diff --git a/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/mod.rs b/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/mod.rs deleted file mode 100644 index 3dad18cb7a3..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/mod.rs +++ /dev/null @@ -1,197 +0,0 @@ -//! A `CustomLogger` that ships finished events to the LiteLLM Python proxy's -//! `/v1/rust_control_plane/logs` endpoint. -//! -//! The callback path is non-blocking: `async_log_success_event` / -//! `async_log_failure_event` -//! build a `LogRecord` and `try_send` it onto a bounded channel, returning a -//! `LogError` (never panicking, never awaiting) if the channel is full or the -//! worker has gone away. A spawned background worker drains the channel, batches -//! records into `{"records":[...]}`, and POSTs them to the proxy with a pooled -//! `reqwest::Client`. - -use std::sync::Arc; -use std::time::Duration; - -use reqwest::Client; -use tokio::sync::mpsc::{self, Receiver, Sender}; -use tokio::time::interval; - -use crate::constants::{DEFAULT_PROXY_BASE_URL, RUST_CONTROL_PLANE_LOGS_PATH}; -use crate::integrations::custom_logger::{ - CallbackTiming, CallbackValue, CustomLogger, LogError, LogFuture, LoggingError, - ModelCallDetails, -}; -use types::{CallbackLogsRequest, EgressTunables, LogRecord}; - -pub mod types; - -/// Ships realtime logging events to the LiteLLM Python proxy. -pub struct LiteLLMPythonProxyAPILogger { - sink: Sender, -} - -impl LiteLLMPythonProxyAPILogger { - /// Spawn the background worker and return a logger handle. `base` is the - /// proxy base URL (no trailing path); `master_key` is sent as a bearer token. - pub fn start(base: String, master_key: String) -> Arc { - let tunables = EgressTunables::from_env(); - let (sink, receiver) = mpsc::channel::(tunables.channel_capacity); - let url = format!( - "{}{}", - base.trim_end_matches('/'), - RUST_CONTROL_PLANE_LOGS_PATH - ); - let client = Client::new(); - tokio::spawn(worker_loop( - receiver, - client, - url, - master_key, - tunables.max_batch_size, - tunables.flush_interval, - )); - Arc::new(Self { sink }) - } - - /// Build a logger from the environment: `LITELLM_PROXY_BASE_URL` (default - /// `http://localhost:4000`) and `LITELLM_MASTER_KEY`. - /// - /// `LITELLM_PROXY_BASE_URL` is treated as the full base and the route is - /// appended verbatim, so if the proxy runs under a `SERVER_ROOT_PATH` - /// (e.g. served at `https://host/litellm`), include it in the base - /// (`LITELLM_PROXY_BASE_URL=https://host/litellm`) and the POST lands at - /// `https://host/litellm/v1/rust_control_plane/logs`. - pub fn from_env() -> Arc { - let base = std::env::var("LITELLM_PROXY_BASE_URL") - .ok() - .filter(|value| !value.trim().is_empty()) - .unwrap_or_else(|| DEFAULT_PROXY_BASE_URL.to_string()); - let key = std::env::var("LITELLM_MASTER_KEY").unwrap_or_default(); - Self::start(base, key) - } - - fn enqueue(&self, record: LogRecord) -> Result<(), LogError> { - self.sink.try_send(record).map_err(|err| match err { - mpsc::error::TrySendError::Full(_) => LogError::channel_full(), - mpsc::error::TrySendError::Closed(_) => LogError::channel_closed(), - }) - } -} - -impl CustomLogger for LiteLLMPythonProxyAPILogger { - fn async_log_success_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - _response_obj: &'a CallbackValue, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - if let Some(payload) = &model_call_details.standard_logging_payload { - self.enqueue(LogRecord { - status: "success".to_string(), - payload: payload.clone(), - error: None, - })?; - } - Ok(()) - }) - } - - fn async_log_failure_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - _response_obj: Option<&'a CallbackValue>, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - if let Some(payload) = &model_call_details.standard_logging_payload { - let fallback_error; - let error = match &model_call_details.failure_error { - Some(error) => error, - None => { - fallback_error = LoggingError { - message: "callback failure event".to_string(), - kind: "CallbackFailure".to_string(), - }; - &fallback_error - } - }; - self.enqueue(LogRecord { - status: "failure".to_string(), - payload: payload.clone(), - error: Some(format!("{}: {}", error.kind, error.message)), - })?; - } - Ok(()) - }) - } -} - -/// Drain the channel, batching records and POSTing them to the proxy. Exits when -/// the channel is closed (all senders dropped) and drained. -async fn worker_loop( - mut receiver: Receiver, - client: Client, - url: String, - master_key: String, - max_batch_size: usize, - flush_interval: Duration, -) { - let mut ticker = interval(flush_interval); - let mut batch: Vec = Vec::with_capacity(max_batch_size); - - loop { - tokio::select! { - maybe_record = receiver.recv() => { - match maybe_record { - Some(record) => { - batch.push(record); - if batch.len() >= max_batch_size { - flush(&client, &url, &master_key, &mut batch).await; - } - } - None => { - // Channel closed: flush remaining and exit. - flush(&client, &url, &master_key, &mut batch).await; - break; - } - } - } - _ = ticker.tick() => { - flush(&client, &url, &master_key, &mut batch).await; - } - } - } -} - -/// POST the current batch (if any), clearing it. Errors are logged, not fatal. -async fn flush(client: &Client, url: &str, master_key: &str, batch: &mut Vec) { - if batch.is_empty() { - return; - } - let records = std::mem::take(batch) - .into_iter() - .map(LogRecord::into_callback_record) - .collect(); - let body = CallbackLogsRequest { records }; - - let response = client - .post(url) - .bearer_auth(master_key) - .json(&body) - .send() - .await; - - match response { - Ok(resp) if resp.status().is_success() => {} - Ok(resp) => { - eprintln!( - "litellm-ai-gateway: callback logs POST returned {} to {url}", - resp.status() - ); - } - Err(err) => { - eprintln!("litellm-ai-gateway: callback logs POST failed to {url}: {err}"); - } - } -} diff --git a/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/types.rs b/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/types.rs deleted file mode 100644 index 481a437747f..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/litellm_python_proxy_api/types.rs +++ /dev/null @@ -1,72 +0,0 @@ -use std::time::Duration; - -use serde::Serialize; - -use crate::constants::{ - DEFAULT_CHANNEL_CAPACITY, DEFAULT_FLUSH_INTERVAL_MS, DEFAULT_MAX_BATCH_SIZE, -}; -use crate::integrations::types::StandardLoggingPayload; - -#[derive(Serialize)] -pub struct CallbackLogsRequest { - pub records: Vec, -} - -#[derive(Serialize)] -pub struct CallbackLogRecord { - pub status: String, - pub standard_logging_payload: StandardLoggingPayload, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -#[derive(Clone, Debug)] -pub struct LogRecord { - pub status: String, - pub payload: StandardLoggingPayload, - pub error: Option, -} - -impl LogRecord { - pub fn into_callback_record(self) -> CallbackLogRecord { - CallbackLogRecord { - status: self.status, - standard_logging_payload: self.payload, - error: self.error, - } - } -} - -pub(super) struct EgressTunables { - pub channel_capacity: usize, - pub max_batch_size: usize, - pub flush_interval: Duration, -} - -impl EgressTunables { - pub fn from_env() -> Self { - Self { - channel_capacity: env_positive( - "LITELLM_LOG_CHANNEL_CAPACITY", - DEFAULT_CHANNEL_CAPACITY, - ), - max_batch_size: env_positive("LITELLM_LOG_BATCH_SIZE", DEFAULT_MAX_BATCH_SIZE), - flush_interval: Duration::from_millis(env_positive( - "LITELLM_LOG_FLUSH_INTERVAL_MS", - DEFAULT_FLUSH_INTERVAL_MS, - )), - } - } -} - -fn env_positive(name: &str, default: T) -> T -where - T: std::str::FromStr + PartialOrd + From, -{ - let zero = T::from(0u8); - std::env::var(name) - .ok() - .and_then(|value| value.trim().parse::().ok()) - .filter(|n| *n > zero) - .unwrap_or(default) -} diff --git a/litellm-rust/crates/ai-gateway/src/integrations/mod.rs b/litellm-rust/crates/ai-gateway/src/integrations/mod.rs deleted file mode 100644 index c62f1821ef8..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! Pure-Rust logging integrations. Names map 1:1 to Python -//! `litellm/integrations/`: -//! - [`custom_guardrail::CustomGuardrail`] — the guardrail callback trait -//! - [`custom_logger::CustomLogger`] — the callback trait -//! - [`litellm_python_proxy_api::LiteLLMPythonProxyAPILogger`] — ships events -//! to the Python proxy's `/v1/rust_control_plane/logs` endpoint -//! - [`types`] — the typed `StandardLoggingPayload` wire contract - -pub mod custom_guardrail; -pub mod custom_logger; -pub mod litellm_python_proxy_api; -pub mod types; diff --git a/litellm-rust/crates/ai-gateway/src/integrations/types.rs b/litellm-rust/crates/ai-gateway/src/integrations/types.rs deleted file mode 100644 index 34dce93d8e0..00000000000 --- a/litellm-rust/crates/ai-gateway/src/integrations/types.rs +++ /dev/null @@ -1,83 +0,0 @@ -//! Typed payloads for the LiteLLM `/v1/callbacks/logs` realtime-logging contract. -//! -//! Field names below are the EXACT JSON keys the Python replay path + spend-logs -//! builder read. Note the deliberate mix: -//! - `startTime` / `endTime` are camelCase (epoch f64 seconds) -//! - `response_cost` / `prompt_tokens` / etc. are snake_case -//! -//! Mirrors Python `litellm/integrations/` + the proxy `CallbackLogsRequest` -//! contract 1:1. - -use serde::Serialize; -use serde_json::Value; -use std::collections::HashMap; - -/// Cumulative token usage for a realtime session. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct Usage { - pub prompt_tokens: u64, - pub completion_tokens: u64, - pub total_tokens: u64, -} - -/// Cost-attribution metadata threaded from the authenticated request. -#[derive(Clone, Debug, Default)] -pub struct RequestMetadata { - pub user_api_key_hash: Option, - pub user_api_key_user_id: Option, - pub user_api_key_team_id: Option, -} - -/// The self-describing payload. Field names are the EXACT JSON keys the Python -/// replay path + spend-logs builder read. -#[derive(Clone, Debug, Serialize)] -pub struct StandardLoggingPayload { - pub id: String, - pub litellm_call_id: String, - - /// e.g. "realtime", "acompletion". Falls back to "acompletion" if absent. - pub call_type: String, - - pub model: String, - pub custom_llm_provider: String, - - /// Spend ($) written to LiteLLM_SpendLogs.spend. - pub response_cost: f64, - - pub prompt_tokens: u64, - pub completion_tokens: u64, - pub total_tokens: u64, - - /// EPOCH SECONDS as float — camelCase keys, NOT snake_case. - #[serde(rename = "startTime")] - pub start_time: f64, - #[serde(rename = "endTime")] - pub end_time: f64, - - pub stream: bool, - - pub metadata: StandardLoggingMetadata, - - /// Optional; stored as request input on the spend log row. - #[serde(skip_serializing_if = "Option::is_none")] - pub messages: Option, -} - -/// Cost-attribution keys. The replayer maps these into litellm_params.metadata, -/// which the spend-logs builder reads to set user / team_id / organization_id. -#[derive(Clone, Debug, Serialize, Default)] -pub struct StandardLoggingMetadata { - pub user_api_key_hash: Option, // -> SpendLogs.api_key - pub user_api_key_user_id: Option, // -> SpendLogs.user - pub user_api_key_team_id: Option, // -> SpendLogs.team_id - - // Optional but read by the builder; include when known: - #[serde(skip_serializing_if = "Option::is_none")] - pub user_api_key_alias: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub user_api_key_org_id: Option, // -> SpendLogs.organization_id - #[serde(skip_serializing_if = "Option::is_none")] - pub user_api_key_end_user_id: Option, // -> SpendLogs.end_user - #[serde(skip_serializing_if = "Option::is_none")] - pub spend_logs_metadata: Option>, -} diff --git a/litellm-rust/crates/ai-gateway/src/io/audio_transcription.rs b/litellm-rust/crates/ai-gateway/src/io/audio_transcription.rs deleted file mode 100644 index 80d9e401a5f..00000000000 --- a/litellm-rust/crates/ai-gateway/src/io/audio_transcription.rs +++ /dev/null @@ -1 +0,0 @@ -pub use crate::audio_transcription::{AudioTranscriptionRequest, audio_transcription}; diff --git a/litellm-rust/crates/ai-gateway/src/io/mod.rs b/litellm-rust/crates/ai-gateway/src/io/mod.rs deleted file mode 100644 index 7098d67993f..00000000000 --- a/litellm-rust/crates/ai-gateway/src/io/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub mod audio_transcription; -pub mod ocr; -pub mod realtime; -pub mod realtime_pool; -pub mod responses_ws; -pub(crate) mod tls; diff --git a/litellm-rust/crates/ai-gateway/src/io/ocr.rs b/litellm-rust/crates/ai-gateway/src/io/ocr.rs deleted file mode 100644 index 2fc82f0b61f..00000000000 --- a/litellm-rust/crates/ai-gateway/src/io/ocr.rs +++ /dev/null @@ -1 +0,0 @@ -pub use crate::ocr::{OcrRequest, ocr}; diff --git a/litellm-rust/crates/ai-gateway/src/io/realtime.rs b/litellm-rust/crates/ai-gateway/src/io/realtime.rs deleted file mode 100644 index 1aa31adcc38..00000000000 --- a/litellm-rust/crates/ai-gateway/src/io/realtime.rs +++ /dev/null @@ -1,418 +0,0 @@ -//! End-to-end OpenAI realtime invocation. -//! -//! The host-facing entry point opens the WebSocket to OpenAI, then splices a -//! client realtime stream to the upstream, driving typed events through the pure -//! `OPENAI_REALTIME_CONFIG` transforms. -//! Network, auth header, key resolution, and wire (de)serialization live here so -//! the `transformation` module stays pure and typed. -//! -//! The dial and splice steps are factored out ([`dial_upstream`], [`splice`]) so -//! the connection pool ([`crate::io::realtime_pool`]) can pre-establish an upstream, -//! buffer its `session.created`, and later hand the live socket to the same -//! splice loop a fresh dial uses. - -use std::time::Duration; - -use futures_util::stream::{SplitSink, SplitStream}; -use futures_util::{Sink, SinkExt, Stream, StreamExt}; -use litellm_core::AuthError; -use litellm_core::auth::error::MissingCredential; -use litellm_core::error::Error; -use litellm_core::realtime::transformation::RealtimeProviderConfig; -use litellm_core::realtime::types::RealtimeEvent; -use tokio::net::TcpStream; -use tokio_tungstenite::tungstenite::Message; -use tokio_tungstenite::tungstenite::client::IntoClientRequest; -use tokio_tungstenite::tungstenite::http::HeaderValue; -use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION; -use tokio_tungstenite::{MaybeTlsStream, WebSocketStream}; - -use litellm_core::providers::openai::realtime::transformation::OPENAI_REALTIME_CONFIG; - -use crate::io::tls::connect_upstream; - -/// Environment variable holding the OpenAI API key (last-resort fallback). -const OPENAI_API_KEY_ENV: &str = "OPENAI_API_KEY"; - -/// Default **idle** timeout: if neither side sends a frame for this long, the -/// session is reaped. It resets on any activity, so it does not cap a healthy -/// (continuously streaming) session — it only frees a stalled one (e.g. a -/// half-open upstream that keeps the socket open but stops sending). -const DEFAULT_IDLE_TIMEOUT_SECS: u64 = 300; - -/// The concrete upstream WebSocket type (TLS or plain). Shared by the dial path -/// and the pool so warm sockets and fresh sockets are the exact same type. -pub type UpstreamWs = WebSocketStream>; -pub(crate) type UpstreamTx = SplitSink; -pub(crate) type UpstreamRx = SplitStream; - -/// Resolve the OpenAI API key from the explicit param or the environment. -/// -/// Blank/whitespace values are treated as absent (guard at resolution time). -pub(crate) fn resolve_api_key(api_key: Option<&str>) -> Result { - api_key - .map(str::trim) - .filter(|key| !key.is_empty()) - .map(str::to_string) - .or_else(|| { - std::env::var(OPENAI_API_KEY_ENV) - .ok() - .filter(|key| !key.trim().is_empty()) - }) - .ok_or_else(|| Error::from(AuthError::from(MissingCredential::OpenAiRealtimeApiKey))) -} - -/// Open the upstream WebSocket to OpenAI for `(model, api_key, api_base)`. -/// -/// This is the dial half of [`realtime`], factored out so the pool can -/// pre-establish sockets ahead of any client. `api_key` here is already resolved -/// (non-blank) — the pool resolves it once when it is created. -pub(crate) async fn dial_upstream( - model: &str, - api_key: &str, - api_base: Option<&str>, -) -> Result { - let url = OPENAI_REALTIME_CONFIG.complete_url(api_base, model); - - let mut request = url - .as_str() - .into_client_request() - .map_err(|err| Error::Network(err.to_string()))?; - // GA realtime: only Authorization. The legacy OpenAI-Beta header triggers - // beta_api_shape_disabled, so we do not send it. - request.headers_mut().insert( - AUTHORIZATION, - HeaderValue::from_str(&format!("Bearer {api_key}")) - .map_err(|err| Error::Auth(err.to_string()))?, - ); - - let (upstream, _response) = connect_upstream(request) - .await - .map_err(|err| Error::Network(err.to_string()))?; - Ok(upstream) -} - -/// Read the next text frame from the upstream and decode it as a typed event. -/// -/// Used by the pool to pre-read OpenAI's unprompted `session.created`. Returns an -/// error on a non-text frame, a closed socket, or undecodable JSON so the pool can -/// discard a misbehaving socket rather than warm it. -pub(crate) async fn read_event(upstream_rx: &mut UpstreamRx) -> Result { - loop { - let message = upstream_rx - .next() - .await - .ok_or_else(|| Error::Network("upstream closed before first event".to_string()))? - .map_err(|err| Error::Network(err.to_string()))?; - match message { - Message::Text(text) => { - return serde_json::from_str(&text) - .map_err(|err| Error::InvalidResponse(err.to_string())); - } - // Ignore protocol frames (ping/pong) while waiting for the first event. - Message::Ping(_) | Message::Pong(_) => continue, - Message::Close(_) => { - return Err(Error::Network( - "upstream closed before first event".to_string(), - )); - } - _ => continue, - } - } -} - -/// Splice an already-connected upstream to the client streams. -/// -/// `prelude` is relayed to the client first (the pool passes the buffered -/// `session.created` here; the fresh-dial path passes `None` and lets the upstream -/// deliver it). Then a single select loop forwards both directions through the -/// transforms until either side closes or the idle timeout fires. -/// `observe` is invoked on **upstream→client** events only (the trusted side that -/// carries `session.created` and `response.done` usage) — never on client events, -/// so a client cannot fabricate usage into its own logs. -#[allow(clippy::too_many_arguments)] -pub(crate) async fn splice( - model: &str, - mut upstream_tx: UpstreamTx, - mut upstream_rx: UpstreamRx, - prelude: Option, - idle_timeout: Option, - mut observe: impl FnMut(&RealtimeEvent) + Send, - mut client_in: In, - mut client_out: Out, -) -> Result<(), Error> -where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - >::Error: std::fmt::Display, -{ - let config = &OPENAI_REALTIME_CONFIG; - - // Relay a buffered backend event (warm handoff's session.created) first, so a - // warm session looks identical to a fresh one from the client's view. - if let Some(event) = prelude { - for outbound in config.transform_realtime_response(&event, model)?.events { - client_out - .send(outbound) - .await - .map_err(|err| Error::Network(err.to_string()))?; - } - } - - let idle = idle_timeout.unwrap_or(Duration::from_secs(DEFAULT_IDLE_TIMEOUT_SECS)); - - // One loop forwarding both directions. The `sleep(idle)` arm is rebuilt every - // iteration, so any frame (either way) resets it — it fires only when the - // session has been fully idle for `idle`, reaping a stalled connection - // (task + upstream TCP socket) instead of leaking it. - loop { - tokio::select! { - // client -> upstream - client_event = client_in.next() => { - let Some(event) = client_event else { break }; // client disconnected - // NOTE: do NOT observe client events. session.created / response.done - // (carrying usage) are server→client events; observing the client arm - // would let an authenticated client POST a fabricated response.done and - // inflate its own spend log. Logging observes upstream events only. - for outbound in config.transform_realtime_request(&event, model)?.events { - let payload = serde_json::to_string(&outbound) - .map_err(|err| Error::InvalidResponse(err.to_string()))?; - upstream_tx - .send(Message::Text(payload)) - .await - .map_err(|err| Error::Network(err.to_string()))?; - } - } - // upstream -> client - upstream_message = upstream_rx.next() => { - let Some(message) = upstream_message else { break }; // upstream closed - match message.map_err(|err| Error::Network(err.to_string()))? { - Message::Text(text) => { - let event: RealtimeEvent = serde_json::from_str(&text) - .map_err(|err| Error::InvalidResponse(err.to_string()))?; - observe(&event); - for outbound in config.transform_realtime_response(&event, model)?.events { - client_out - .send(outbound) - .await - .map_err(|err| Error::Network(err.to_string()))?; - } - } - Message::Close(_) => break, - _ => {} - } - } - // idle timeout: no activity from either side within `idle` - _ = tokio::time::sleep(idle) => break, - } - } - Ok(()) -} - -/// Splice a client realtime stream to OpenAI: forward client events upstream -/// (via `transform_realtime_request`) and backend events downstream (via -/// `transform_realtime_response`). Returns when either side closes. -/// -/// Generic over the client transport (typed events) so this crate stays -/// framework-agnostic; the gateway adapts its axum socket to these. This is the -/// fresh-dial path: dial, then splice. The pool's warm-handoff path skips the dial -/// and calls [`splice`] directly with a buffered `session.created`. -#[allow(clippy::too_many_arguments)] -pub async fn realtime( - model: &str, - api_key: Option<&str>, - api_base: Option<&str>, - idle_timeout: Option, - observe: impl FnMut(&RealtimeEvent) + Send, - client_in: In, - client_out: Out, -) -> Result<(), Error> -where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - >::Error: std::fmt::Display, -{ - let api_key = resolve_api_key(api_key)?; - let upstream = dial_upstream(model, &api_key, api_base).await?; - let (upstream_tx, upstream_rx) = upstream.split(); - splice( - model, - upstream_tx, - upstream_rx, - None, - idle_timeout, - observe, - client_in, - client_out, - ) - .await -} - -/// Splice a pre-warmed upstream (taken from [`crate::io::realtime_pool`]) to the -/// client. Relays the buffered `session.created` first, then splices exactly like -/// the fresh-dial path — so a warm session is indistinguishable from a fresh one. -#[allow(clippy::too_many_arguments)] -pub async fn realtime_warm( - model: &str, - handoff: crate::io::realtime_pool::WarmHandoff, - idle_timeout: Option, - observe: impl FnMut(&RealtimeEvent) + Send, - client_in: In, - client_out: Out, -) -> Result<(), Error> -where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - >::Error: std::fmt::Display, -{ - splice( - model, - handoff.tx, - handoff.rx, - Some(handoff.session_created), - idle_timeout, - observe, - client_in, - client_out, - ) - .await -} - -#[cfg(test)] -mod tests { - use super::*; - - fn event(raw: &str) -> RealtimeEvent { - serde_json::from_str(raw).expect("valid event json") - } - - /// The realtime dial has to reach a `wss://` upstream without a process-wide - /// crypto provider installed, which is what dialing through `io::tls` buys. - #[tokio::test] - async fn dial_upstream_over_wss_reports_an_error_instead_of_panicking() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind a loopback port"); - let port = listener - .local_addr() - .expect("read the bound address") - .port(); - tokio::spawn(async move { - while let Ok((stream, _peer)) = listener.accept().await { - drop(stream); - } - }); - - let result = dial_upstream( - "gpt-realtime", - "sk-test", - Some(&format!("wss://127.0.0.1:{port}")), - ) - .await; - - assert!(matches!(result, Err(Error::Network(_)))); - } - - #[test] - fn resolve_api_key_prefers_param_then_blank_falls_through() { - assert_eq!(resolve_api_key(Some("sk-test")).unwrap(), "sk-test"); - // A blank param with no env set should error. - if std::env::var(OPENAI_API_KEY_ENV).is_err() { - assert!(resolve_api_key(Some(" ")).is_err()); - } - } - - /// Live end-to-end check against OpenAI. Ignored by default (CI never runs - /// it); run explicitly with `OPENAI_API_KEY` set: - /// `cargo test -p litellm-ai-gateway --features server realtime_invokes_openai -- --ignored --nocapture` - #[tokio::test] - #[ignore = "hits the live OpenAI realtime API; needs OPENAI_API_KEY"] - async fn realtime_invokes_openai_and_responds() { - use futures_channel::mpsc; - - let key = - std::env::var(OPENAI_API_KEY_ENV).expect("set OPENAI_API_KEY to run this ignored test"); - - // client -> provider (we hold `client_tx` to push events upstream) - let (mut client_tx, client_in) = mpsc::unbounded::(); - // provider -> client (we hold `backend_rx` to read backend events) - let (client_out, mut backend_rx) = mpsc::unbounded::(); - - // Clone the key so the spawned task owns its `String` (no borrow across await). - let key_owned = key.clone(); - let call = tokio::spawn(async move { - realtime( - "gpt-realtime", - Some(&key_owned), - None, - None, - |_| {}, - client_in, - client_out, - ) - .await - }); - - // 1. First backend event should be session.created. - let first = tokio::time::timeout(Duration::from_secs(30), backend_rx.next()) - .await - .expect("timed out waiting for session.created") - .expect("backend stream closed before session.created"); - assert_eq!( - first.event_type, "session.created", - "expected session.created, got: {}", - first.event_type - ); - - // 2. Ask for a short audio response. - client_tx - .send(event( - r#"{"type":"conversation.item.create","item":{"type":"message","role":"user","content":[{"type":"input_text","text":"Say hi."}]}}"#, - )) - .await - .expect("send conversation.item.create"); - client_tx - .send(event(r#"{"type":"response.create"}"#)) - .await - .expect("send response.create"); - - // 3. Read backend events; require a non-empty audio delta, then response.done. - let mut saw_audio_delta = false; - let mut saw_done = false; - for _ in 0..500 { - let next = tokio::time::timeout(Duration::from_secs(30), backend_rx.next()).await; - let event = match next { - Ok(Some(event)) => event, - Ok(None) => break, - Err(_) => panic!("timed out waiting for backend events"), - }; - match event.event_type.as_str() { - "response.output_audio.delta" => { - let delta = event - .data - .get("delta") - .and_then(|value| value.as_str()) - .unwrap_or(""); - if !delta.is_empty() { - saw_audio_delta = true; - } - } - "response.done" => { - saw_done = true; - break; - } - _ => {} - } - } - - assert!( - saw_audio_delta, - "expected a response.output_audio.delta with non-empty delta" - ); - assert!(saw_done, "expected a response.done event"); - - // Drop the client sender so the provider's to_upstream side finishes. - drop(client_tx); - let _ = call.await; - } -} diff --git a/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs b/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs deleted file mode 100644 index 49e9c459a88..00000000000 --- a/litellm-rust/crates/ai-gateway/src/io/realtime_pool.rs +++ /dev/null @@ -1,712 +0,0 @@ -//! Pre-warmed upstream realtime connection pool. -//! -//! The gateway's realtime overhead lives entirely in session establishment: on -//! every client connect it dials a fresh upstream WS to OpenAI and waits for -//! `session.created` before it can serve. This pool keeps a small set of upstream -//! sockets **already connected and already past `session.created`** so a connect -//! can be served from a warm socket and the handshake is off the critical path. -//! -//! Layering: this lives in the gateway's `io` module next to the dial/splice it -//! reuses. The gateway holds an `Arc` in its state and asks for a -//! warm socket per connect; on a miss it fresh-dials exactly as before. The pool -//! is a latency optimization, never a correctness dependency — see the gateway's -//! `src/routes/realtime/README.md`. -//! -//! ## Caveats (enforced here) -//! - One warm socket serves exactly one session (realtime isn't multiplexed), so -//! the pool is sized to the connect *rate*, not concurrent connections. -//! - `session.created` is pre-read once and buffered; nothing else is read from a -//! warm socket before handoff, so a warm session starts at OpenAI defaults just -//! like a fresh one (`session.update` semantics unchanged). -//! - Warm sockets are short-lived (`max_idle`) and liveness-checked at handoff to -//! bound idle billing / dodge OpenAI's idle timeout. -//! - On miss or dead socket the caller fresh-dials; the pool never blocks or fails -//! a connect because it is empty. - -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; -use std::time::{Duration, Instant}; - -use futures_util::StreamExt; -use litellm_core::Error; -use litellm_core::realtime::types::RealtimeEvent; - -use crate::io::realtime::{ - UpstreamRx, UpstreamTx, UpstreamWs, dial_upstream, read_event, resolve_api_key, -}; - -/// Default target warm sockets per key when pooling is enabled. -pub const DEFAULT_POOL_SIZE: usize = 4; - -/// Default max time a warm socket may sit before it is closed and replaced. -pub const DEFAULT_MAX_IDLE: Duration = Duration::from_secs(30); - -/// Env var: target warm sockets per key. `0` disables pooling (fresh-dial only). -pub const POOL_SIZE_ENV: &str = "REALTIME_POOL_SIZE"; - -/// Env var: max warm-socket idle lifetime, in seconds. -pub const MAX_IDLE_ENV: &str = "REALTIME_POOL_MAX_IDLE_SECS"; - -/// How often the background replenisher wakes to top up and reap stale sockets. -const REPLENISH_TICK: Duration = Duration::from_millis(250); - -/// Backoff floor after a key's warm-up dials all fail. The first failed pass -/// waits this long before retrying that key. -const BACKOFF_BASE: Duration = Duration::from_millis(500); - -/// Backoff ceiling. A key that keeps failing (invalid credentials, an -/// unreachable upstream) is retried at most once per this interval — instead of -/// firing `needed` concurrent TLS dials every 250 ms tick, which would hammer -/// the upstream and risk rate-limit exhaustion that degrades valid cold-path -/// traffic. Backoff resets the moment a dial for the key succeeds. -const BACKOFF_MAX: Duration = Duration::from_secs(30); - -/// Identifies an upstream connection: the tuple that fully determines the dial. -/// `api_key` is included so a warm socket is only ever reused for the same key -/// (no cross-tenant reuse). -#[derive(Clone, PartialEq, Eq, Hash)] -pub struct UpstreamKey { - pub model: String, - pub api_key: String, - pub api_base: Option, -} - -impl std::fmt::Debug for UpstreamKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("UpstreamKey") - .field("model", &self.model) - .field("api_key", &"[REDACTED]") - .field("api_base", &self.api_base) - .finish() - } -} - -/// A warm upstream: split halves + the buffered `session.created` + when it was -/// warmed (for `max_idle` expiry). -struct WarmConnection { - tx: UpstreamTx, - rx: UpstreamRx, - session_created: RealtimeEvent, - warmed_at: Instant, -} - -/// A live upstream taken from the pool, ready to splice. The caller relays -/// `session_created` to the client first, then splices `(tx, rx)` as usual. -pub struct WarmHandoff { - pub tx: UpstreamTx, - pub rx: UpstreamRx, - pub session_created: RealtimeEvent, -} - -/// Pool configuration, resolved once at startup from the environment. -#[derive(Clone, Copy, Debug)] -pub struct PoolConfig { - /// Target warm sockets per key. `0` disables pooling. - pub target_size: usize, - /// Max time a warm socket may sit before it is closed and replaced. - pub max_idle: Duration, -} - -impl Default for PoolConfig { - fn default() -> Self { - Self { - target_size: DEFAULT_POOL_SIZE, - max_idle: DEFAULT_MAX_IDLE, - } - } -} - -impl PoolConfig { - /// Read config from the environment, falling back to defaults. An invalid - /// value warns and uses the default rather than failing startup. - pub fn from_env() -> Self { - let target_size = match std::env::var(POOL_SIZE_ENV) { - Ok(raw) => raw.trim().parse().unwrap_or_else(|_| { - eprintln!("warning: {POOL_SIZE_ENV}={raw:?} is not a valid size; using {DEFAULT_POOL_SIZE}"); - DEFAULT_POOL_SIZE - }), - Err(_) => DEFAULT_POOL_SIZE, - }; - let max_idle = match std::env::var(MAX_IDLE_ENV) { - Ok(raw) => raw - .trim() - .parse() - .map(Duration::from_secs) - .unwrap_or_else(|_| { - eprintln!( - "warning: {MAX_IDLE_ENV}={raw:?} is not a valid number of seconds; using {}s", - DEFAULT_MAX_IDLE.as_secs() - ); - DEFAULT_MAX_IDLE - }), - Err(_) => DEFAULT_MAX_IDLE, - }; - Self { - target_size, - max_idle, - } - } - - /// Whether pooling is on (`target_size > 0`). - pub fn enabled(&self) -> bool { - self.target_size > 0 - } -} - -/// Per-key warm sockets, behind a single `Mutex`. Realtime warm sockets are few -/// (the pool is small), so a plain mutex over a `VecDeque`-ish `Vec` is simpler -/// and faster than sharding; contention is negligible at this scale. -type Warm = HashMap>; - -/// Per-key replenish backoff. Absent (or `consecutive_failures == 0`) means the -/// key is healthy and replenished every tick. After a pass whose dials all fail, -/// `retry_after` is pushed out with exponential backoff so a broken key (invalid -/// credentials, unreachable upstream) is not re-dialed on every 250 ms tick. -#[derive(Default)] -struct Backoff { - /// Don't attempt warm-up dials for this key until this instant. `None` = - /// eligible now. - retry_after: Option, - consecutive_failures: u32, -} - -type Backoffs = HashMap; - -/// Pre-warmed upstream realtime connection pool. -/// -/// Cheap to clone-via-`Arc`. The background replenisher is spawned by -/// [`RealtimePool::spawn`]; a pool built with [`RealtimePool::disabled`] never -/// warms anything and every `take` misses (callers fresh-dial). -pub struct RealtimePool { - config: PoolConfig, - warm: Mutex, - /// Per-key replenish backoff so a broken key doesn't trigger unbounded - /// concurrent dials every tick. Separate lock from `warm` so the request - /// hot path (`take`) never contends on it. - backoff: Mutex, -} - -impl RealtimePool { - /// A disabled pool: no background task, every `take` returns `None`. - pub fn disabled() -> Arc { - Arc::new(Self { - config: PoolConfig { - target_size: 0, - ..PoolConfig::default() - }, - warm: Mutex::new(HashMap::new()), - backoff: Mutex::new(HashMap::new()), - }) - } - - /// Build a pool from config **without** the background replenisher. The pool - /// only warms when [`RealtimePool::warm_now`] is called. Used by deterministic - /// unit tests; production uses [`RealtimePool::spawn`]. - #[cfg(test)] - fn new_unspawned(config: PoolConfig) -> Arc { - Arc::new(Self { - config, - warm: Mutex::new(HashMap::new()), - backoff: Mutex::new(HashMap::new()), - }) - } - - /// Build a pool from config and, if enabled, spawn the background replenisher. - /// Returns the shared handle the gateway stores in its state. - pub fn spawn(config: PoolConfig) -> Arc { - let pool = Arc::new(Self { - config, - warm: Mutex::new(HashMap::new()), - backoff: Mutex::new(HashMap::new()), - }); - if config.enabled() { - let weak = Arc::downgrade(&pool); - tokio::spawn(async move { - let mut tick = tokio::time::interval(REPLENISH_TICK); - tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - loop { - tick.tick().await; - // Stop once the gateway has dropped its handle. - let Some(pool) = weak.upgrade() else { break }; - pool.replenish_all().await; - } - }); - } - pool - } - - /// Resolved config (test/inspection). - pub fn config(&self) -> PoolConfig { - self.config - } - - /// Register a key so the replenisher starts warming it. Idempotent. The - /// gateway calls this once per known deployment at startup; the pool only - /// warms keys it has seen, so it never dials a model nobody asked for. - pub fn register(&self, key: UpstreamKey) { - if !self.config.enabled() { - return; - } - self.warm.lock().unwrap().entry(key).or_default(); - } - - /// Take a warm, live socket for `key`, or `None` on miss / dead socket. - /// - /// Pops the freshest non-expired socket and liveness-checks it; a socket that - /// is too old or already dead is dropped (closing it) and the next candidate - /// tried. Never blocks: if nothing warm is live, returns `None` so the caller - /// fresh-dials. - pub fn take(&self, key: &UpstreamKey) -> Option { - if !self.config.enabled() { - return None; - } - loop { - let mut candidate = { - let mut warm = self.warm.lock().unwrap(); - let bucket = warm.get_mut(key)?; - bucket.pop()? - }; - // Discard sockets past their warm lifetime (idle-billing guard). - if candidate.warmed_at.elapsed() > self.config.max_idle { - continue; // drops `candidate`, closing the socket - } - // Liveness: a non-blocking check that the socket hasn't already - // delivered a Close/Err. A warm socket should be silent after - // session.created, so anything pending means it is unhealthy. - if is_dead(&mut candidate.rx) { - continue; - } - return Some(WarmHandoff { - tx: candidate.tx, - rx: candidate.rx, - session_created: candidate.session_created, - }); - } - } - - /// One replenish pass over every registered key: reap stale sockets, then - /// dial up to `target_size`. Dials run concurrently; failures are swallowed - /// (a key that can't be warmed just keeps fresh-dialing on the request path) - /// and put the key into exponential backoff so a broken key isn't re-dialed - /// on every tick. - async fn replenish_all(&self) { - let keys: Vec = { self.warm.lock().unwrap().keys().cloned().collect() }; - for key in keys { - self.reap_stale(&key); - // Skip keys still in backoff from a prior all-failed pass — this is - // what bounds dials against an invalid/unreachable key to once per - // `BACKOFF_MAX` instead of `needed` dials every 250 ms tick. - if self.in_backoff(&key) { - continue; - } - let needed = { - let warm = self.warm.lock().unwrap(); - let have = warm.get(&key).map(Vec::len).unwrap_or(0); - self.config.target_size.saturating_sub(have) - }; - if needed == 0 { - continue; - } - // Dial the missing sockets CONCURRENTLY. A sequential loop here makes - // a full refill cost `needed × handshake` (~needed × 350 ms), which - // can't keep up with a high connect rate — the pool drains faster - // than it refills and most connects miss. Firing the dials together - // refills in ~one handshake window, keeping warm supply ≈ peak - // concurrent connects so the sub-ms warm handoff becomes the median, - // not the lucky-hit tail. - let dials = (0..needed).map(|_| warm_one(&key)); - let results = futures_util::future::join_all(dials).await; - let mut any_ok = false; - // `.flatten()` keeps only the successful dials; a key that can't be - // warmed just keeps fresh-dialing on the request path. - for conn in results.into_iter().flatten() { - any_ok = true; - self.warm - .lock() - .unwrap() - .entry(key.clone()) - .or_default() - .push(conn); - } - // Reset backoff on any success; otherwise grow it. We only ever enter - // backoff when a pass that *attempted* dials produced none — a `needed - // == 0` pass is handled by the `continue` above and never touches it. - self.record_replenish_outcome(&key, any_ok); - } - } - - /// Whether `key` is currently in a backoff window (a prior pass failed and - /// the retry time hasn't arrived). Eligible keys are pruned from the backoff - /// map so it doesn't grow unbounded for healthy keys. - fn in_backoff(&self, key: &UpstreamKey) -> bool { - let mut backoff = self.backoff.lock().unwrap(); - match backoff.get(key).and_then(|b| b.retry_after) { - Some(retry_after) if Instant::now() < retry_after => true, - Some(_) => { - // Window elapsed — allow the attempt. Keep the failure count so a - // still-broken key backs off further, but clear the gate so this - // tick proceeds. - if let Some(b) = backoff.get_mut(key) { - b.retry_after = None; - } - false - } - None => false, - } - } - - /// Update a key's backoff after a replenish attempt. Success clears it; - /// failure grows the retry delay exponentially up to `BACKOFF_MAX`. - fn record_replenish_outcome(&self, key: &UpstreamKey, any_ok: bool) { - let mut backoff = self.backoff.lock().unwrap(); - if any_ok { - backoff.remove(key); - return; - } - let entry = backoff.entry(key.clone()).or_default(); - entry.consecutive_failures = entry.consecutive_failures.saturating_add(1); - // Exponential: BASE * 2^(failures-1), saturating at MAX. `min` of the - // shift exponent keeps the doubling from overflowing. - let shift = (entry.consecutive_failures - 1).min(16); - let delay = BACKOFF_BASE.saturating_mul(1u32 << shift).min(BACKOFF_MAX); - entry.retry_after = Some(Instant::now() + delay); - } - - /// Drop sockets past `max_idle` or already dead for a key. - fn reap_stale(&self, key: &UpstreamKey) { - let mut warm = self.warm.lock().unwrap(); - if let Some(bucket) = warm.get_mut(key) { - bucket.retain_mut(|conn| { - conn.warmed_at.elapsed() <= self.config.max_idle && !is_dead(&mut conn.rx) - }); - } - } - - /// Test/inspection: number of warm sockets currently held for `key`. - #[cfg(test)] - pub fn warm_len(&self, key: &UpstreamKey) -> usize { - self.warm - .lock() - .unwrap() - .get(key) - .map(Vec::len) - .unwrap_or(0) - } - - /// Test/inspection: consecutive replenish failures recorded for `key` (0 if - /// the key is healthy / has no backoff entry). - #[cfg(test)] - pub fn backoff_failures(&self, key: &UpstreamKey) -> u32 { - self.backoff - .lock() - .unwrap() - .get(key) - .map(|b| b.consecutive_failures) - .unwrap_or(0) - } - - /// Test helper: synchronously warm `target_size` sockets for `key` (no - /// background task). Lets tests assert handoff behavior deterministically. - #[cfg(test)] - pub async fn warm_now(&self, key: &UpstreamKey) { - let needed = { - let warm = self.warm.lock().unwrap(); - let have = warm.get(key).map(Vec::len).unwrap_or(0); - self.config.target_size.saturating_sub(have) - }; - for _ in 0..needed { - if let Ok(conn) = warm_one(key).await { - self.warm - .lock() - .unwrap() - .entry(key.clone()) - .or_default() - .push(conn); - } - } - } - - /// Test helper: insert an already-built warm connection (used to inject a - /// dead socket and assert it is discarded at handoff). - #[cfg(test)] - fn insert_warm(&self, key: UpstreamKey, conn: WarmConnection) { - self.warm.lock().unwrap().entry(key).or_default().push(conn); - } -} - -/// Dial one upstream and pre-read its `session.created` into a [`WarmConnection`]. -/// -/// `key.api_key` is already resolved (non-blank). The first frame OpenAI sends -/// unprompted is `session.created`; we buffer exactly that and read nothing more. -async fn warm_one(key: &UpstreamKey) -> Result { - let upstream: UpstreamWs = - dial_upstream(&key.model, &key.api_key, key.api_base.as_deref()).await?; - let (tx, mut rx) = upstream.split(); - let session_created = read_event(&mut rx).await?; - Ok(WarmConnection { - tx, - rx, - session_created, - warmed_at: Instant::now(), - }) -} - -/// Resolve a deployment's API key into the pool key, returning `None` when no key -/// can be resolved (those deployments simply aren't pooled — the request path -/// still fresh-dials and surfaces the auth error there). -pub fn upstream_key( - model: &str, - api_key: Option<&str>, - api_base: Option<&str>, -) -> Option { - let api_key = resolve_api_key(api_key).ok()?; - Some(UpstreamKey { - model: model.to_string(), - api_key, - api_base: api_base.map(str::to_string), - }) -} - -/// Non-blocking liveness check: poll the upstream once. A warm socket is silent -/// after `session.created`, so a pending `Close`/`Err`/`None` means it is dead. -/// A pending data frame (shouldn't happen pre-handoff) is also treated as -/// unhealthy — we'd rather discard and fresh-dial than hand over a socket in an -/// unexpected state. `Pending` (the healthy case) returns `false`. -fn is_dead(rx: &mut UpstreamRx) -> bool { - use futures_util::Stream; - use futures_util::task::noop_waker_ref; - use std::pin::Pin; - use std::task::{Context, Poll}; - - let mut cx = Context::from_waker(noop_waker_ref()); - match Pin::new(rx).poll_next(&mut cx) { - Poll::Pending => false, - Poll::Ready(None) => true, - Poll::Ready(Some(Err(_))) => true, - // Any frame arriving before handoff is unexpected for a silent warm - // socket; treat it as unhealthy. - Poll::Ready(Some(Ok(_))) => true, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use futures_util::SinkExt; - use std::net::SocketAddr; - use tokio::net::TcpListener; - use tokio_tungstenite::tungstenite::Message; - - /// An in-process fake OpenAI realtime WS server. On connect it sends - /// `session.created`; on `response.create` it sends `response.created` + - /// `response.output_audio.delta` + `response.done`. Returns its `ws://` base. - async fn spawn_fake_openai() -> String { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr: SocketAddr = listener.local_addr().unwrap(); - tokio::spawn(async move { - while let Ok((stream, _)) = listener.accept().await { - tokio::spawn(handle_fake_conn(stream)); - } - }); - format!("ws://{addr}") - } - - async fn handle_fake_conn(stream: tokio::net::TcpStream) { - let mut ws = match tokio_tungstenite::accept_async(stream).await { - Ok(ws) => ws, - Err(_) => return, - }; - // Unprompted session.created, exactly like OpenAI. - let _ = ws - .send(Message::Text( - r#"{"type":"session.created","session":{"id":"sess_fake"}}"#.to_string(), - )) - .await; - while let Some(Ok(msg)) = ws.next().await { - if let Message::Text(text) = msg - && text.contains("response.create") - { - for frame in [ - r#"{"type":"response.created"}"#, - r#"{"type":"response.output_audio.delta","delta":"AAAA"}"#, - r#"{"type":"response.done"}"#, - ] { - let _ = ws.send(Message::Text(frame.to_string())).await; - } - } - } - } - - fn test_config() -> PoolConfig { - PoolConfig { - target_size: 2, - max_idle: Duration::from_secs(30), - } - } - - fn key_for(base: &str) -> UpstreamKey { - UpstreamKey { - model: "gpt-realtime".to_string(), - api_key: "sk-test".to_string(), - api_base: Some(base.to_string()), - } - } - - #[tokio::test] - async fn warm_handoff_relays_buffered_session_created() { - let base = spawn_fake_openai().await; - let pool = RealtimePool::new_unspawned(test_config()); - let key = key_for(&base); - pool.register(key.clone()); - pool.warm_now(&key).await; - assert_eq!(pool.warm_len(&key), 2); - - let handoff = pool.take(&key).expect("a warm socket should be available"); - assert_eq!(handoff.session_created.event_type, "session.created"); - assert_eq!( - handoff - .session_created - .data - .get("session") - .and_then(|s| s.get("id")) - .and_then(|v| v.as_str()), - Some("sess_fake") - ); - // Taking one leaves one. - assert_eq!(pool.warm_len(&key), 1); - } - - #[tokio::test] - async fn pool_miss_returns_none_for_fresh_dial_fallback() { - let base = spawn_fake_openai().await; - let pool = RealtimePool::new_unspawned(test_config()); - let key = key_for(&base); - // Registered but never warmed → empty bucket → miss. - pool.register(key.clone()); - assert!(pool.take(&key).is_none()); - - // Unknown key → miss. - let other = key_for("ws://127.0.0.1:1"); - assert!(pool.take(&other).is_none()); - } - - #[tokio::test] - async fn disabled_pool_never_hands_off() { - let pool = RealtimePool::disabled(); - let key = key_for("ws://127.0.0.1:1"); - pool.register(key.clone()); - assert_eq!(pool.warm_len(&key), 0); - assert!(pool.take(&key).is_none()); - } - - #[tokio::test] - async fn dead_warm_socket_is_discarded() { - let base = spawn_fake_openai().await; - let pool = RealtimePool::new_unspawned(test_config()); - let key = key_for(&base); - pool.register(key.clone()); - - // Build one real warm connection, then kill the upstream by dropping the - // server side: easiest is to dial, read session.created, then close our - // own rx's peer. Instead we forge "dead" via an already-closed socket: - // dial a connection and immediately send a Close from the client side so - // the server closes back, then warm it. Simpler: warm normally, then - // mark it stale by backdating warmed_at past max_idle and confirm it's - // dropped — that exercises the same discard path. - let mut conn = warm_one(&key).await.expect("warm one"); - conn.warmed_at = Instant::now() - Duration::from_secs(3600); // past max_idle - pool.insert_warm(key.clone(), conn); - assert_eq!(pool.warm_len(&key), 1); - - // take() must discard the stale socket and report a miss. - assert!(pool.take(&key).is_none()); - assert_eq!(pool.warm_len(&key), 0); - } - - #[tokio::test] - async fn background_replenisher_tops_up_registered_key() { - let base = spawn_fake_openai().await; - let pool = RealtimePool::spawn(test_config()); - let key = key_for(&base); - pool.register(key.clone()); - - // Wait (bounded) for the background task to reach the target size. - let mut warmed = 0; - for _ in 0..40 { - tokio::time::sleep(Duration::from_millis(50)).await; - warmed = pool.warm_len(&key); - if warmed >= test_config().target_size { - break; - } - } - assert_eq!( - warmed, - test_config().target_size, - "background replenisher should warm up to target_size" - ); - let handoff = pool.take(&key).expect("a warm socket should be available"); - assert_eq!(handoff.session_created.event_type, "session.created"); - } - - #[tokio::test] - async fn closed_upstream_socket_is_detected_dead() { - // A genuinely dead socket: dial the fake, read session.created, then drop - // the server by closing from our side and waiting for the close to land. - let base = spawn_fake_openai().await; - let pool = RealtimePool::new_unspawned(test_config()); - let key = key_for(&base); - pool.register(key.clone()); - - let mut conn = warm_one(&key).await.expect("warm one"); - // Close the upstream from the client side; the server echoes a close. - let _ = conn.tx.send(Message::Close(None)).await; - // Give the close a moment to arrive on rx. - tokio::time::sleep(Duration::from_millis(50)).await; - pool.insert_warm(key.clone(), conn); - - // Liveness check at take() should detect the close and discard it. - assert!(pool.take(&key).is_none()); - assert_eq!(pool.warm_len(&key), 0); - } - - #[tokio::test] - async fn broken_key_backs_off_instead_of_dialing_every_tick() { - // A key whose upstream is unreachable: every warm-up dial fails. - let pool = RealtimePool::new_unspawned(test_config()); - let key = key_for("ws://127.0.0.1:1"); // nothing listens here - pool.register(key.clone()); - - // First pass attempts dials, they all fail → key enters backoff, no warm - // sockets, one recorded failure. - pool.replenish_all().await; - assert_eq!(pool.warm_len(&key), 0); - assert_eq!(pool.backoff_failures(&key), 1); - assert!( - pool.in_backoff(&key), - "a key whose dials all failed must be in backoff" - ); - - // An immediate next pass must be SKIPPED (still in the backoff window), so - // it does NOT fire another round of dials — the failure count is unchanged. - pool.replenish_all().await; - assert_eq!( - pool.backoff_failures(&key), - 1, - "replenish during the backoff window must not re-dial the broken key" - ); - } - - #[tokio::test] - async fn healthy_key_never_enters_backoff_and_clears_after_recovery() { - let base = spawn_fake_openai().await; - let pool = RealtimePool::new_unspawned(test_config()); - let key = key_for(&base); - pool.register(key.clone()); - - // A reachable upstream: the pass succeeds, so the key is never backed off. - pool.replenish_all().await; - assert_eq!(pool.warm_len(&key), test_config().target_size); - assert_eq!(pool.backoff_failures(&key), 0); - assert!(!pool.in_backoff(&key)); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs b/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs deleted file mode 100644 index f86dd778424..00000000000 --- a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs +++ /dev/null @@ -1,485 +0,0 @@ -use std::time::Duration; - -use futures_util::stream::{SplitSink, SplitStream}; -use futures_util::{Sink, SinkExt, Stream, StreamExt}; -use litellm_core::AuthError; -use litellm_core::Error; -use litellm_core::auth::error::MissingCredential; -use litellm_core::providers::openai::responses::transformation::OPENAI_RESPONSES_WS_CONFIG; -use litellm_core::responses::types::ResponsesWsEvent; -use litellm_core::responses::websocket::ResponsesWebSocketProviderConfig; -use tokio_tungstenite::tungstenite::Message; -use tokio_tungstenite::tungstenite::client::IntoClientRequest; -use tokio_tungstenite::tungstenite::http::HeaderValue; -use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION; - -use litellm_core::responses::websocket::{ResponsesUpstreamWs, connect_upstream}; - -use crate::constants::{ - DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS, DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS, -}; - -const OPENAI_API_KEY_ENV: &str = "OPENAI_API_KEY"; -type UpstreamTx = SplitSink; -type UpstreamRx = SplitStream; - -pub(crate) fn resolve_api_key(api_key: Option<&str>) -> Result { - api_key - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) - .or_else(|| { - std::env::var(OPENAI_API_KEY_ENV) - .ok() - .filter(|value| !value.trim().is_empty()) - }) - .ok_or_else(|| Error::from(AuthError::from(MissingCredential::OpenAiResponsesApiKey))) -} - -async fn dial_upstream( - model: &str, - api_key: &str, - api_base: Option<&str>, -) -> Result { - let url = OPENAI_RESPONSES_WS_CONFIG.complete_websocket_url(api_base, model); - let mut request = url - .as_str() - .into_client_request() - .map_err(|error| Error::Network(error.to_string()))?; - request.headers_mut().insert( - AUTHORIZATION, - HeaderValue::from_str(&format!("Bearer {api_key}")) - .map_err(|error| Error::Auth(error.to_string()))?, - ); - let result = tokio::time::timeout( - Duration::from_secs(DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS), - connect_upstream(request), - ) - .await - .map_err(|_| Error::Network("Responses WebSocket connection timed out".to_string()))?; - result - .map(|(socket, _)| socket) - .map_err(|error| match *error { - tokio_tungstenite::tungstenite::Error::Http(response) => Error::Http { - status: response.status().as_u16(), - body: String::new(), - }, - other => Error::Network(other.to_string()), - }) -} - -pub struct ResponsesWebSocketStreaming; - -impl ResponsesWebSocketStreaming { - pub async fn bidirectional_forward( - model: &str, - upstream_tx: UpstreamTx, - upstream_rx: UpstreamRx, - idle_timeout: Option, - observe: impl FnMut(&ResponsesWsEvent) + Send, - client_in: In, - client_out: Out, - ) -> Result<(), Error> - where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - Out::Error: std::fmt::Display, - { - splice( - model, - upstream_tx, - upstream_rx, - idle_timeout, - observe, - client_in, - client_out, - ) - .await - } -} - -pub(crate) async fn splice( - model: &str, - mut upstream_tx: UpstreamTx, - mut upstream_rx: UpstreamRx, - idle_timeout: Option, - mut observe: impl FnMut(&ResponsesWsEvent) + Send, - mut client_in: In, - mut client_out: Out, -) -> Result<(), Error> -where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - Out::Error: std::fmt::Display, -{ - let idle = - idle_timeout.unwrap_or_else(|| Duration::from_secs(DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS)); - loop { - tokio::select! { - event = client_in.next() => { - let Some(event) = event else { break }; - for outbound in OPENAI_RESPONSES_WS_CONFIG - .transform_ws_request(&event, model)? - .events - { - let payload = serde_json::to_string(&outbound) - .map_err(|error| Error::InvalidResponse(error.to_string()))?; - upstream_tx.send(Message::Text(payload)) - .await - .map_err(|error| Error::Network(error.to_string()))?; - } - } - message = upstream_rx.next() => { - let Some(message) = message else { break }; - match message.map_err(|error| Error::Network(error.to_string()))? { - Message::Text(text) => { - let event = serde_json::from_str::(&text) - .map_err(|error| Error::InvalidResponse(error.to_string()))?; - observe(&event); - for outbound in OPENAI_RESPONSES_WS_CONFIG - .transform_ws_response(&event, model)? - .events - { - client_out.send(outbound) - .await - .map_err(|error| Error::Network(error.to_string()))?; - } - } - Message::Close(_) => break, - _ => {} - } - } - _ = tokio::time::sleep(idle) => break, - } - } - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -pub async fn async_responses_websocket( - model: &str, - api_key: Option<&str>, - api_base: Option<&str>, - first_frame: Option, - idle_timeout: Option, - mut observe: impl FnMut(&ResponsesWsEvent) + Send, - client_in: In, - client_out: Out, -) -> Result<(), Error> -where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - Out::Error: std::fmt::Display, -{ - let key = resolve_api_key(api_key)?; - let upstream = dial_upstream(model, &key, api_base).await?; - let (mut upstream_tx, upstream_rx) = upstream.split(); - if let Some(first_frame) = first_frame { - for outbound in OPENAI_RESPONSES_WS_CONFIG - .transform_ws_request(&first_frame, model)? - .events - { - let payload = serde_json::to_string(&outbound) - .map_err(|error| Error::InvalidResponse(error.to_string()))?; - upstream_tx - .send(Message::Text(payload)) - .await - .map_err(|error| Error::Network(error.to_string()))?; - } - } - ResponsesWebSocketStreaming::bidirectional_forward( - model, - upstream_tx, - upstream_rx, - idle_timeout, - &mut observe, - client_in, - client_out, - ) - .await -} - -#[allow(clippy::too_many_arguments)] -pub async fn responses_ws( - model: &str, - api_key: Option<&str>, - api_base: Option<&str>, - first_frame: Option, - idle_timeout: Option, - observe: impl FnMut(&ResponsesWsEvent) + Send, - client_in: In, - client_out: Out, -) -> Result<(), Error> -where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - Out::Error: std::fmt::Display, -{ - async_responses_websocket( - model, - api_key, - api_base, - first_frame, - idle_timeout, - observe, - client_in, - client_out, - ) - .await -} - -#[cfg(test)] -mod tests { - use super::*; - use futures_channel::mpsc; - use futures_util::{SinkExt, StreamExt}; - use litellm_core::responses::types::ResponsesWsEventType; - use serde_json::json; - use tokio::io::AsyncWriteExt; - use tokio::net::TcpListener; - use tokio_tungstenite::accept_async; - - /// The Responses dial has to reach a `wss://` upstream without a process-wide - /// crypto provider installed, which is what dialing through `io::tls` buys. - #[tokio::test] - async fn dial_upstream_over_wss_reports_an_error_instead_of_panicking() { - let listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("bind a loopback port"); - let port = listener - .local_addr() - .expect("read the bound address") - .port(); - tokio::spawn(async move { - while let Ok((stream, _peer)) = listener.accept().await { - drop(stream); - } - }); - - let result = - dial_upstream("gpt-5", "sk-test", Some(&format!("wss://127.0.0.1:{port}"))).await; - - assert!(matches!(result, Err(Error::Network(_)))); - } - - async fn websocket_base() -> (String, tokio::task::JoinHandle<()>) { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let address = listener.local_addr().expect("local address"); - let task = tokio::spawn(async move { - let (stream, _) = listener.accept().await.expect("accept"); - let mut socket = accept_async(stream).await.expect("websocket handshake"); - while let Some(Ok(Message::Text(text))) = socket.next().await { - let request: serde_json::Value = serde_json::from_str(&text).expect("request json"); - let model = request - .get("model") - .and_then(serde_json::Value::as_str) - .or_else(|| { - request - .get("response") - .and_then(serde_json::Value::as_object) - .and_then(|response| { - response.get("model").and_then(serde_json::Value::as_str) - }) - }) - .expect("enforced model"); - socket - .send(Message::Text( - json!({ - "type": "response.created", - "response": { - "id": format!("resp-{model}"), - "model": model, - "extra": "preserved" - } - }) - .to_string(), - )) - .await - .expect("created event"); - socket - .send(Message::Text( - json!({ - "type": "response.completed", - "response": { - "id": format!("resp-{model}"), - "model": model, - "usage": { - "input_tokens": 1, - "output_tokens": 2, - "total_tokens": 3 - } - } - }) - .to_string(), - )) - .await - .expect("completed event"); - } - }); - (format!("http://{address}"), task) - } - - fn event(value: serde_json::Value) -> ResponsesWsEvent { - serde_json::from_value(value).expect("event") - } - - #[test] - fn explicit_nonblank_key_wins() { - assert_eq!( - resolve_api_key(Some(" explicit ")).expect("key"), - "explicit" - ); - } - - #[test] - fn blank_key_is_not_accepted_without_environment_key() { - if std::env::var(OPENAI_API_KEY_ENV).is_err() { - assert!(resolve_api_key(Some(" ")).is_err()); - } - } - - #[tokio::test] - async fn forwards_events_sequentially_and_enforces_model() { - let (api_base, server) = websocket_base().await; - let (client_tx, client_rx) = mpsc::unbounded(); - let (output_tx, mut output_rx) = mpsc::unbounded(); - let (observed_tx, observed_rx) = mpsc::unbounded(); - client_tx - .unbounded_send(event(json!({ - "type": "response.create", - "model": "wrong" - }))) - .expect("first request"); - client_tx - .unbounded_send(event(json!({ - "type": "response.create", - "response": {"model": "also-wrong"} - }))) - .expect("second request"); - - let task = tokio::spawn(async move { - responses_ws( - "authorized-model", - Some("test-key"), - Some(&api_base), - None, - Some(Duration::from_secs(1)), - move |event| { - observed_tx - .unbounded_send(event.clone()) - .expect("observe event"); - }, - client_rx, - output_tx, - ) - .await - }); - - let first = output_rx.next().await.expect("first output"); - let second = output_rx.next().await.expect("second output"); - let third = output_rx.next().await.expect("third output"); - let fourth = output_rx.next().await.expect("fourth output"); - drop(client_tx); - task.await.expect("splice task").expect("successful splice"); - server.await.expect("server task"); - - assert_eq!(first.event_type, ResponsesWsEventType::ResponseCreated); - assert_eq!(first.model(), Some("authorized-model")); - assert_eq!(first.data["response"]["extra"], "preserved"); - assert_eq!(second.event_type, ResponsesWsEventType::ResponseCompleted); - assert_eq!(third.event_type, ResponsesWsEventType::ResponseCreated); - assert_eq!(fourth.event_type, ResponsesWsEventType::ResponseCompleted); - let observed: Vec<_> = observed_rx.collect().await; - assert_eq!(observed.len(), 4); - assert!( - observed - .iter() - .all(|event| event.event_type != ResponsesWsEventType::ResponseCreate) - ); - } - - #[tokio::test] - async fn idle_timeout_ends_without_upstream_events() { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let address = listener.local_addr().expect("address"); - let server = tokio::spawn(async move { - let (stream, _) = listener.accept().await.expect("accept"); - let _socket = accept_async(stream).await.expect("handshake"); - tokio::time::sleep(Duration::from_secs(1)).await; - }); - let (_client_tx, client_rx) = mpsc::unbounded::(); - let (output_tx, mut output_rx) = mpsc::unbounded(); - let result = responses_ws( - "model", - Some("key"), - Some(&format!("http://{address}")), - None, - Some(Duration::from_millis(20)), - |_| {}, - client_rx, - output_tx, - ) - .await; - assert!(result.is_ok()); - assert!(output_rx.next().await.is_none()); - server.abort(); - } - - #[tokio::test] - async fn dial_http_status_is_preserved() { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let address = listener.local_addr().expect("address"); - let server = tokio::spawn(async move { - let (mut stream, _) = listener.accept().await.expect("accept"); - stream - .write_all(b"HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\n\r\n") - .await - .expect("response"); - }); - let (_client_tx, client_rx) = mpsc::unbounded::(); - let (output_tx, _output_rx) = mpsc::unbounded(); - let error = responses_ws( - "model", - Some("key"), - Some(&format!("http://{address}")), - None, - Some(Duration::from_millis(20)), - |_| {}, - client_rx, - output_tx, - ) - .await - .expect_err("status error"); - assert!(matches!(error, Error::Http { status: 401, .. })); - server.await.expect("server task"); - } - - #[tokio::test] - async fn dial_http_500_status_is_preserved() { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let address = listener.local_addr().expect("address"); - let server = tokio::spawn(async move { - let (mut stream, _) = listener.accept().await.expect("accept"); - stream - .write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n") - .await - .expect("response"); - }); - let (_client_tx, client_rx) = mpsc::unbounded::(); - let (output_tx, _output_rx) = mpsc::unbounded(); - let error = responses_ws( - "model", - Some("key"), - Some(&format!("http://{address}")), - None, - Some(Duration::from_millis(20)), - |_| {}, - client_rx, - output_tx, - ) - .await - .expect_err("status error"); - assert!(matches!(error, Error::Http { status: 500, .. })); - server.await.expect("server task"); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/io/tls.rs b/litellm-rust/crates/ai-gateway/src/io/tls.rs deleted file mode 100644 index a2562f60345..00000000000 --- a/litellm-rust/crates/ai-gateway/src/io/tls.rs +++ /dev/null @@ -1,80 +0,0 @@ -//! Outbound WebSocket dials over a TLS config this crate builds once and owns. -//! -//! `reqwest/rustls-tls` enables `rustls/ring` and `litellm-core`'s `bedrock-auth` -//! enables `rustls/aws-lc-rs`, so the bare `ClientConfig::builder()` that -//! `tokio-tungstenite` uses when handed no connector panics rather than guess -//! between them. Naming ring on a connector of our own settles that for these -//! dials without touching the process-wide default, and building the config -//! once keeps the platform trust store, which `tokio-tungstenite` would -//! otherwise re-read on every dial, off the dial path. - -use std::io; -use std::sync::{Arc, OnceLock}; - -use rustls::{ClientConfig, RootCertStore}; -use tokio::net::TcpStream; -use tokio_tungstenite::tungstenite::Error; -use tokio_tungstenite::tungstenite::client::IntoClientRequest; -use tokio_tungstenite::tungstenite::error::TlsError; -use tokio_tungstenite::tungstenite::handshake::client::Response; -use tokio_tungstenite::{ - Connector, MaybeTlsStream, WebSocketStream, connect_async_tls_with_config, -}; - -static TLS_CONFIG: OnceLock> = OnceLock::new(); - -fn build_config() -> Result> { - let native = rustls_native_certs::load_native_certs(); - let roots = { - let mut store = RootCertStore::empty(); - let (added, _ignored) = store.add_parsable_certificates(native.certs); - if added == 0 { - return Err(Box::new(Error::Io(io::Error::other(format!( - "no usable native root certificates: {:?}", - native.errors - ))))); - } - store - }; - - ClientConfig::builder_with_provider(Arc::new(rustls::crypto::ring::default_provider())) - .with_safe_default_protocol_versions() - .map(|builder| builder.with_root_certificates(roots).with_no_client_auth()) - .map_err(|error| Box::new(Error::Tls(TlsError::Rustls(error)))) -} - -fn tls_config() -> Result, Box> { - if let Some(config) = TLS_CONFIG.get() { - return Ok(Arc::clone(config)); - } - let built = Arc::new(build_config()?); - Ok(Arc::clone(TLS_CONFIG.get_or_init(|| built))) -} - -pub(crate) async fn connect_upstream( - request: R, -) -> Result<(WebSocketStream>, Response), Box> -where - R: IntoClientRequest + Unpin, -{ - let request = request.into_client_request().map_err(Box::new)?; - let connector = match request.uri().scheme_str() { - Some("wss") => Some(Connector::Rustls(tls_config()?)), - _ => None, - }; - connect_async_tls_with_config(request, None, false, connector) - .await - .map_err(Box::new) -} - -#[cfg(test)] -mod tests { - use super::build_config; - - #[test] - fn builds_a_usable_config_with_both_provider_features_enabled() { - let config = build_config().expect("a client config"); - - assert!(!config.crypto_provider().cipher_suites.is_empty()); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/lib.rs b/litellm-rust/crates/ai-gateway/src/lib.rs deleted file mode 100644 index 08fbde564ed..00000000000 --- a/litellm-rust/crates/ai-gateway/src/lib.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! LiteLLM AI Gateway library. -//! -//! Two layers, split by feature so the Python `cdylib` can depend on the I/O -//! without pulling in the HTTP server: -//! -//! - Call-type modules such as [`ocr`]: provider transforms, lifecycle hooks, -//! and provider I/O. Always available — no feature required. These predate the -//! rule that a route's entrypoint and handler live in `litellm-core` (see -//! `litellm_core::messages`) and move there as they are touched. -//! - [`io`]: compatibility exports and realtime WebSocket splice helpers. -//! - The server modules ([`auth`], [`routes`], [`state`]) and anything pulling -//! `axum` are gated behind the `server` feature, which the `litellm-ai-gateway` -//! binary turns on. - -pub mod audio_transcription; -mod client; -pub mod io; -pub mod ocr; - -#[cfg(feature = "server")] -pub mod auth; -#[cfg(feature = "server")] -pub mod routes; -#[cfg(feature = "server")] -pub mod state; -#[cfg(feature = "trace-parity")] -pub mod trace_parity; - -mod constants; -pub mod integrations; -#[cfg(feature = "server")] -mod realtime; diff --git a/litellm-rust/crates/ai-gateway/src/main.rs b/litellm-rust/crates/ai-gateway/src/main.rs deleted file mode 100644 index 88d7b1dbcf8..00000000000 --- a/litellm-rust/crates/ai-gateway/src/main.rs +++ /dev/null @@ -1,162 +0,0 @@ -//! LiteLLM AI Gateway — a minimal Axum server fronting the Rust router. -//! -//! Flow: client → `POST /v1/realtime` → `router.realtime()` selects a deployment -//! (simple-shuffle) → `io::realtime::realtime()` invokes OpenAI. The -//! server owns transport + config; routing lives in the `router` crate. -//! -//! The binary requires the `server` feature (declared in `Cargo.toml` via -//! `required-features`), so cargo skips it unless that feature is on. Everything -//! the binary needs lives in the library (`litellm_ai_gateway`); `main` just -//! wires startup. - -use std::sync::Arc; - -use litellm_ai_gateway::io::realtime_pool::{PoolConfig, RealtimePool, upstream_key}; -use litellm_ai_gateway::routes; -use litellm_ai_gateway::state::AppState; -#[cfg(feature = "python-config")] -use litellm_config::load_model_list; -use litellm_core::router::{Deployment, LiteLLMParams, Router}; - -use litellm_ai_gateway::integrations::custom_logger::CustomLogger; -use litellm_ai_gateway::integrations::litellm_python_proxy_api::LiteLLMPythonProxyAPILogger; - -/// Bind to localhost by default so the gateway is not a public, unauthenticated -/// provider proxy out of the box. Override with `HOST` (e.g. `0.0.0.0`). -const DEFAULT_HOST: &str = "127.0.0.1"; -const DEFAULT_PORT: u16 = 4001; - -#[tokio::main] -async fn main() { - // Trim before storing so it matches the trimmed bearer token in `auth` - // (avoids a silent auth failure when the env var has surrounding whitespace). - let master_key: Option> = std::env::var("LITELLM_MASTER_KEY") - .ok() - .map(|key| key.trim().to_string()) - .filter(|key| !key.is_empty()) - .map(Arc::from); - if master_key.is_none() { - eprintln!( - "warning: LITELLM_MASTER_KEY is not set; /v1/realtime will reject all requests (fail closed)" - ); - } - - // Spawn the realtime-logging worker (drains a channel → POSTs batches to the - // Python proxy's /v1/callbacks/logs). Built here so the spawn lands on the - // tokio runtime. `from_env` reads LITELLM_PROXY_BASE_URL + LITELLM_MASTER_KEY. - let proxy_logger = LiteLLMPythonProxyAPILogger::from_env(); - let loggers: Vec> = vec![proxy_logger]; - - let router = Arc::new(build_router()); - - // Build the pre-warmed realtime pool and register each deployment's upstream - // so the background replenisher starts warming it. `REALTIME_POOL_SIZE=0` - // yields a disabled pool → every connect fresh-dials (original behavior). - let pool_config = PoolConfig::from_env(); - let realtime_pool = RealtimePool::spawn(pool_config); - if pool_config.enabled() { - register_deployments(&router, &realtime_pool); - eprintln!( - "realtime connection pool enabled: target {} warm sockets/key, max idle {}s", - pool_config.target_size, - pool_config.max_idle.as_secs() - ); - } else { - eprintln!( - "realtime connection pool disabled (REALTIME_POOL_SIZE=0); fresh-dialing each connect" - ); - } - - let state = AppState { - router, - master_key, - loggers: Arc::new(loggers), - realtime_pool, - }; - - let host = std::env::var("HOST").unwrap_or_else(|_| DEFAULT_HOST.to_string()); - let port = resolve_port(); - - let listener = tokio::net::TcpListener::bind((host.as_str(), port)) - .await - .expect("failed to bind listener"); - eprintln!("litellm-ai-gateway listening on {host}:{port}"); - axum::serve(listener, routes::app(state)) - .await - .expect("server error"); -} - -/// Register every deployment's upstream key with the pool so the replenisher -/// pre-warms it. Mirrors `service::run`'s key derivation (strip `openai/`, resolve -/// api_key); deployments whose key can't be resolved are skipped (they fresh-dial -/// and surface the auth error on the request path, as before). -fn register_deployments(router: &Router, pool: &RealtimePool) { - for deployment in router.deployments() { - let params = &deployment.litellm_params; - let provider_model = params - .model - .strip_prefix("openai/") - .unwrap_or(¶ms.model); - if let Some(key) = upstream_key( - provider_model, - params.api_key.as_deref(), - params.api_base.as_deref(), - ) { - pool.register(key); - } - } -} - -/// Resolve `PORT`, warning (rather than silently defaulting) on an invalid value. -fn resolve_port() -> u16 { - match std::env::var("PORT") { - Ok(raw) => raw.parse().unwrap_or_else(|_| { - eprintln!("warning: PORT={raw:?} is not a valid port; using {DEFAULT_PORT}"); - DEFAULT_PORT - }), - Err(_) => DEFAULT_PORT, - } -} - -/// Build the router. With the `python-config` feature and `LITELLM_CONFIG_PATH` -/// set, load the resolved `model_list` from the proxy config via the embedded -/// Python reader (load time only). Otherwise fall back to the env stand-in. -fn build_router() -> Router { - #[cfg(feature = "python-config")] - if let Ok(config_path) = std::env::var("LITELLM_CONFIG_PATH") { - match load_model_list(std::path::Path::new(&config_path)) { - Ok(deployments) => { - eprintln!("loaded model_list from {config_path} via python config reader"); - return Router::new(deployments); - } - Err(err) => { - eprintln!("config load failed ({err}); falling back to env deployment"); - } - } - } - build_router_from_env() -} - -/// Build a minimal single-deployment `model_list` from the environment. -/// -/// A real deployment loads `model_list` from config; this is the minimal stand-in -/// so the gateway has one OpenAI deployment to route to. -fn build_router_from_env() -> Router { - let model = - std::env::var("OPENAI_REALTIME_MODEL").unwrap_or_else(|_| "gpt-realtime".to_string()); - let api_key = std::env::var("OPENAI_API_KEY").ok(); - if api_key.is_none() { - eprintln!( - "warning: OPENAI_API_KEY is not set; realtime requests will fail with auth errors" - ); - } - let deployment = Deployment { - model_name: model.clone(), - litellm_params: LiteLLMParams { - model, - api_key, - api_base: None, - }, - }; - Router::new(vec![deployment]) -} diff --git a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs b/litellm-rust/crates/ai-gateway/src/ocr/mod.rs deleted file mode 100644 index fb63a02f7ad..00000000000 --- a/litellm-rust/crates/ai-gateway/src/ocr/mod.rs +++ /dev/null @@ -1,127 +0,0 @@ -use litellm_core::Error; -use litellm_core::ocr::{ - OcrClient, - wire::{OcrWireRequest, decode_request}, -}; -use serde_json::Value; - -mod types; - -pub use types::OcrRequest; - -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub async fn ocr(request: OcrRequest<'_>) -> Result { - core_ocr(request).await -} - -async fn core_ocr(request: OcrRequest<'_>) -> Result { - validate_host_hooks(&request)?; - let client = OcrClient::new(crate::client::http_client().clone())?; - let core_request = decode_request(OcrWireRequest { - model: request.model.to_string(), - document: request.document, - api_key: request.api_key.map(str::to_string), - api_base: request.api_base.map(str::to_string), - custom_llm_provider: request.custom_llm_provider.map(str::to_string), - extra_headers: request.extra_headers, - optional_params: request.optional_params, - input_sources: Default::default(), - timeout_seconds: request.timeout.map(|timeout| timeout.as_secs_f64()), - })?; - client - .perform(core_request) - .await - .map(|response| response.into_json()) -} - -fn validate_host_hooks(request: &OcrRequest<'_>) -> Result<(), Error> { - if !request.guardrails.is_empty() { - return Err(Error::Unsupported( - "OCR host guardrails are not wired to the core path", - )); - } - if !request.callbacks.is_empty() { - return Err(Error::Unsupported( - "OCR host callbacks are not wired to the core path", - )); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use litellm_core::ocr::wire::is_supported_request; - use serde_json::{Map, json}; - - use super::{OcrRequest, validate_host_hooks}; - use crate::integrations::custom_guardrail::{CustomGuardrail, GuardrailEventHook}; - use crate::integrations::custom_logger::CustomLogger; - - struct TestGuardrail; - - impl CustomGuardrail for TestGuardrail { - fn guardrail_name(&self) -> &str { - "test" - } - - fn supported_event_hooks(&self) -> &[GuardrailEventHook] { - &[] - } - } - - struct TestLogger; - - impl CustomLogger for TestLogger {} - - fn request() -> OcrRequest<'static> { - OcrRequest { - model: "model", - document: json!({"type":"image_url","image_url":"data:image/png;base64,YQ=="}), - api_key: None, - api_base: None, - custom_llm_provider: Some("mistral"), - extra_headers: None, - optional_params: Map::new(), - timeout: None, - callbacks: Vec::new(), - guardrails: Vec::new(), - request_metadata: Default::default(), - litellm_call_id: None, - } - } - - #[test] - fn core_activation_includes_migrated_providers() { - assert!(is_supported_request("model", Some("mistral"))); - assert!(is_supported_request("pixtral-12b", Some("azure_ai"))); - assert!(is_supported_request( - "doc-intelligence/prebuilt-layout", - Some("azure_ai") - )); - assert!(is_supported_request("parse-v3", Some("reducto"))); - assert!(is_supported_request("mistral-ocr", Some("vertex_ai"))); - assert!(is_supported_request("deepseek-ocr", Some("vertex_ai"))); - } - - #[test] - fn core_path_rejects_unwired_guardrails() { - let request = OcrRequest { - guardrails: vec![Arc::new(TestGuardrail)], - ..request() - }; - let error = validate_host_hooks(&request).unwrap_err(); - assert!(error.to_string().contains("guardrails are not wired")); - } - - #[test] - fn core_path_rejects_unwired_callbacks() { - let request = OcrRequest { - callbacks: vec![Arc::new(TestLogger)], - ..request() - }; - let error = validate_host_hooks(&request).unwrap_err(); - assert!(error.to_string().contains("callbacks are not wired")); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/ocr/types.rs b/litellm-rust/crates/ai-gateway/src/ocr/types.rs deleted file mode 100644 index e96d2df1adb..00000000000 --- a/litellm-rust/crates/ai-gateway/src/ocr/types.rs +++ /dev/null @@ -1,23 +0,0 @@ -use std::sync::Arc; -use std::time::Duration; - -use serde_json::{Map, Value}; - -use crate::integrations::custom_guardrail::CustomGuardrail; -use crate::integrations::custom_logger::CustomLogger; -use crate::integrations::types::RequestMetadata; - -pub struct OcrRequest<'a> { - pub model: &'a str, - pub document: Value, - pub api_key: Option<&'a str>, - pub api_base: Option<&'a str>, - pub custom_llm_provider: Option<&'a str>, - pub extra_headers: Option>, - pub optional_params: Map, - pub timeout: Option, - pub callbacks: Vec>, - pub guardrails: Vec>, - pub request_metadata: RequestMetadata, - pub litellm_call_id: Option<&'a str>, -} diff --git a/litellm-rust/crates/ai-gateway/src/realtime/mod.rs b/litellm-rust/crates/ai-gateway/src/realtime/mod.rs deleted file mode 100644 index 82be596ba86..00000000000 --- a/litellm-rust/crates/ai-gateway/src/realtime/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Realtime logging collector. Observes the realtime event stream and emits a -//! `StandardLoggingPayload` to the registered callbacks on session close. - -pub mod streaming; diff --git a/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs b/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs deleted file mode 100644 index c0d72e90b77..00000000000 --- a/litellm-rust/crates/ai-gateway/src/realtime/streaming.rs +++ /dev/null @@ -1,414 +0,0 @@ -//! `RealTimeStreaming` — the realtime logging collector. -//! -//! Mirrors Python `litellm.realtime_api.main.RealTimeStreaming`: it observes the -//! event stream in O(1) (never buffering frames), accumulating just the fields -//! the spend log needs (model, id, cumulative usage), then on session close -//! builds a `StandardLoggingPayload` and fans it out to every registered -//! `CustomLogger`. - -use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; - -use litellm_core::realtime::types::RealtimeEvent; -use serde_json::Value; - -use crate::constants::DEFAULT_PROVIDER; -use crate::integrations::custom_logger::{ - CallbackTiming, CallbackValue, CustomLogger, CustomLoggerRunner, LoggingError, ModelCallDetails, -}; -use crate::integrations::types::{ - RequestMetadata, StandardLoggingMetadata, StandardLoggingPayload, Usage, -}; - -/// Current wall-clock time as epoch seconds (float), matching the Python -/// `startTime`/`endTime` contract. -fn epoch_seconds() -> f64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs_f64()) - .unwrap_or(0.0) -} - -/// Status of a finished realtime session, mapped to the callback record status. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum SessionStatus { - Success, - Failure, -} - -/// Accumulates realtime session state and emits a logging payload on close. -pub struct RealTimeStreaming { - callbacks: Vec>, - /// REQUEST-ID RULE: the SpendLogs `request_id` == the OpenAI realtime session - /// id (`sess_…`), captured from `session.created`. Both `id` and - /// `litellm_call_id` are set to that value so the Python writer logs the same - /// id regardless of which field it reads. The gateway-generated `rt-…` id - /// (the constructor seed) is only a fallback for sessions that fail before - /// `session.created` arrives. - litellm_call_id: String, - /// See the request-id rule above — mirrors `litellm_call_id`. - id: String, - model: String, - custom_llm_provider: String, - usage: Usage, - response_cost: f64, - start_time: f64, - end_time: f64, - metadata: RequestMetadata, - /// Count of logging callbacks that failed to enqueue (non-fatal). - dropped: u64, -} - -impl RealTimeStreaming { - /// Create a collector for one session. `litellm_call_id` is the gateway's - /// per-connection id; `model` is the requested model (a sane default until - /// `session.created` reports the upstream model). - pub fn new( - callbacks: Vec>, - litellm_call_id: String, - model: String, - metadata: RequestMetadata, - ) -> Self { - let now = epoch_seconds(); - Self { - callbacks, - id: litellm_call_id.clone(), - litellm_call_id, - model, - custom_llm_provider: DEFAULT_PROVIDER.to_string(), - usage: Usage::default(), - response_cost: 0.0, - start_time: now, - end_time: now, - metadata, - dropped: 0, - } - } - - /// Number of logging callbacks that failed to enqueue so far (test/observ.). - #[allow(dead_code)] - pub fn dropped(&self) -> u64 { - self.dropped - } - - /// Observe one realtime event. O(1): updates accumulated state only; never - /// buffers frames. Safe to call on every event in either direction. - pub fn observe(&mut self, event: &RealtimeEvent) { - match event.event_type.as_str() { - "session.created" | "session.updated" => self.on_session(event), - "response.done" => self.on_response_done(event), - _ => {} - } - } - - /// `session.created` / `session.updated` → capture upstream id + model. - /// Per the request-id rule, the OpenAI session id becomes BOTH `id` and - /// `litellm_call_id`, replacing the gateway-generated fallback. - fn on_session(&mut self, event: &RealtimeEvent) { - let session = event.data.get("session").and_then(Value::as_object); - if let Some(id) = session.and_then(|s| s.get("id")).and_then(Value::as_str) - && !id.is_empty() - { - self.id = id.to_string(); - self.litellm_call_id = id.to_string(); - } - if let Some(model) = session.and_then(|s| s.get("model")).and_then(Value::as_str) - && !model.is_empty() - { - self.model = model.to_string(); - } - } - - /// `response.done` → add this response's usage to the cumulative totals. - fn on_response_done(&mut self, event: &RealtimeEvent) { - let usage = event - .data - .get("response") - .and_then(Value::as_object) - .and_then(|r| r.get("usage")) - .and_then(Value::as_object); - let Some(usage) = usage else { return }; - - let input = usage.get("input_tokens").and_then(Value::as_u64); - let output = usage.get("output_tokens").and_then(Value::as_u64); - let total = usage.get("total_tokens").and_then(Value::as_u64); - - if let Some(input) = input { - self.usage.prompt_tokens += input; - } - if let Some(output) = output { - self.usage.completion_tokens += output; - } - // Prefer the upstream-reported total; otherwise derive it. - match total { - Some(total) => self.usage.total_tokens += total, - None => { - self.usage.total_tokens += input.unwrap_or(0) + output.unwrap_or(0); - } - } - } - - /// Set the per-session response cost ($). Cost computation is Python-side in - /// the proxy; the gateway forwards 0.0 by default and lets the proxy price. - /// Public API (exercised in tests) for the future path where the gateway - /// prices realtime sessions itself. - #[allow(dead_code)] - pub fn set_response_cost(&mut self, cost: f64) { - self.response_cost = cost; - } - - /// Build the `StandardLoggingPayload` from accumulated state. - pub fn build_payload(&self) -> StandardLoggingPayload { - StandardLoggingPayload { - id: self.id.clone(), - litellm_call_id: self.litellm_call_id.clone(), - call_type: "realtime".to_string(), - model: self.model.clone(), - custom_llm_provider: self.custom_llm_provider.clone(), - response_cost: self.response_cost, - prompt_tokens: self.usage.prompt_tokens, - completion_tokens: self.usage.completion_tokens, - total_tokens: self.usage.total_tokens, - start_time: self.start_time, - end_time: self.end_time, - stream: true, - metadata: StandardLoggingMetadata { - user_api_key_hash: self.metadata.user_api_key_hash.clone(), - user_api_key_user_id: self.metadata.user_api_key_user_id.clone(), - user_api_key_team_id: self.metadata.user_api_key_team_id.clone(), - ..Default::default() - }, - messages: None, - } - } - - /// Finish the session: stamp the end time and fan the payload out to every - /// callback. On a logger enqueue error we bump a non-fatal counter (the - /// realtime session has already ended; a dropped log must never propagate). - pub async fn log_messages(&mut self, status: SessionStatus) { - self.end_time = epoch_seconds(); - let payload = self.build_payload(); - let timing = CallbackTiming::new(payload.start_time, payload.end_time); - let runner = CustomLoggerRunner::new(self.callbacks.clone()); - - match status { - SessionStatus::Success => { - let response = CallbackValue::new("realtime", serde_json::Value::Null); - let report = runner - .async_log_success_event( - &ModelCallDetails::from_standard_logging_payload(payload), - &response, - timing, - ) - .await; - self.dropped += report.dropped as u64; - } - SessionStatus::Failure => { - let error = LoggingError { - message: "realtime session ended in failure".to_string(), - kind: "RealtimeSessionError".to_string(), - }; - let response = CallbackValue::new( - "error", - serde_json::json!({ - "message": error.message, - "kind": error.kind, - }), - ); - let report = runner - .async_log_failure_event( - &ModelCallDetails::from_standard_logging_payload(payload) - .with_failure_error(error), - Some(&response), - timing, - ) - .await; - self.dropped += report.dropped as u64; - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::integrations::custom_logger::LogError; - use crate::integrations::custom_logger::LogFuture; - use std::sync::atomic::{AtomicU64, Ordering}; - - fn event(raw: &str) -> RealtimeEvent { - serde_json::from_str(raw).expect("valid event json") - } - - /// A test logger that records the last payload it saw. - #[derive(Default)] - struct CapturingLogger { - calls: AtomicU64, - last_model: std::sync::Mutex>, - last_total_tokens: AtomicU64, - } - - impl CustomLogger for CapturingLogger { - fn async_log_success_event<'a>( - &'a self, - model_call_details: &'a ModelCallDetails, - _response_obj: &'a CallbackValue, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async move { - let payload = model_call_details - .standard_logging_payload - .as_ref() - .expect("standard logging payload"); - self.calls.fetch_add(1, Ordering::SeqCst); - *self.last_model.lock().unwrap() = Some(payload.model.clone()); - self.last_total_tokens - .store(payload.total_tokens, Ordering::SeqCst); - Ok(()) - }) - } - } - - #[tokio::test] - async fn observe_accumulates_model_and_tokens_then_logs() { - let logger = Arc::new(CapturingLogger::default()); - let callbacks: Vec> = vec![logger.clone()]; - let mut streaming = RealTimeStreaming::new( - callbacks, - "call_abc".to_string(), - "gpt-realtime".to_string(), - RequestMetadata { - user_api_key_hash: Some("hash123".to_string()), - user_api_key_user_id: Some("user-1".to_string()), - user_api_key_team_id: Some("team-1".to_string()), - }, - ); - - streaming.observe(&event( - r#"{"type":"session.created","session":{"id":"sess_001","model":"gpt-realtime-2025"}}"#, - )); - streaming.observe(&event( - r#"{"type":"response.done","response":{"usage":{"input_tokens":10,"output_tokens":5,"total_tokens":15}}}"#, - )); - // A second response.done accumulates. - streaming.observe(&event( - r#"{"type":"response.done","response":{"usage":{"input_tokens":3,"output_tokens":2,"total_tokens":5}}}"#, - )); - - let payload = streaming.build_payload(); - assert_eq!(payload.model, "gpt-realtime-2025"); - // Request-id rule: session.created's id becomes BOTH id and - // litellm_call_id (replacing the "call_abc" gateway fallback), so the - // SpendLogs request_id is always the OpenAI session id. - assert_eq!(payload.id, "sess_001"); - assert_eq!(payload.litellm_call_id, "sess_001"); - assert_eq!(payload.prompt_tokens, 13); - assert_eq!(payload.completion_tokens, 7); - assert_eq!(payload.total_tokens, 20); - assert_eq!(payload.response_cost, 0.0); - assert_eq!(payload.call_type, "realtime"); - assert_eq!(payload.custom_llm_provider, "openai"); - assert_eq!( - payload.metadata.user_api_key_hash.as_deref(), - Some("hash123") - ); - - streaming.log_messages(SessionStatus::Success).await; - assert_eq!(logger.calls.load(Ordering::SeqCst), 1); - assert_eq!( - logger.last_model.lock().unwrap().as_deref(), - Some("gpt-realtime-2025") - ); - assert_eq!(logger.last_total_tokens.load(Ordering::SeqCst), 20); - assert_eq!(streaming.dropped(), 0); - } - - #[test] - fn blank_session_id_and_model_keep_the_gateway_fallbacks() { - let mut streaming = RealTimeStreaming::new( - Vec::new(), - "call_fallback".to_string(), - "gpt-realtime".to_string(), - RequestMetadata::default(), - ); - - streaming.observe(&event( - r#"{"type":"session.created","session":{"id":"","model":""}}"#, - )); - let payload = streaming.build_payload(); - assert_eq!(payload.id, "call_fallback"); - assert_eq!(payload.litellm_call_id, "call_fallback"); - assert_eq!(payload.model, "gpt-realtime"); - - streaming.observe(&event( - r#"{"type":"session.updated","session":{"id":"sess_002","model":""}}"#, - )); - let payload = streaming.build_payload(); - assert_eq!(payload.id, "sess_002"); - assert_eq!(payload.litellm_call_id, "sess_002"); - assert_eq!(payload.model, "gpt-realtime"); - } - - #[test] - fn payload_serializes_with_camelcase_times_and_realtime_call_type() { - let mut streaming = RealTimeStreaming::new( - Vec::new(), - "call_xyz".to_string(), - "gpt-realtime".to_string(), - RequestMetadata::default(), - ); - streaming.observe(&event( - r#"{"type":"response.done","response":{"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}"#, - )); - streaming.set_response_cost(0.0042); - let payload = streaming.build_payload(); - let json = serde_json::to_string(&payload).expect("serialize payload"); - - assert!(json.contains("\"startTime\""), "missing startTime: {json}"); - assert!(json.contains("\"endTime\""), "missing endTime: {json}"); - assert!( - json.contains("\"call_type\":\"realtime\""), - "missing call_type realtime: {json}" - ); - assert!( - json.contains("\"response_cost\""), - "missing response_cost: {json}" - ); - assert_eq!(payload.response_cost, 0.0042); - } - - /// A logger whose enqueue always fails should bump the dropped counter, not - /// panic or propagate. - #[tokio::test] - async fn failing_logger_bumps_dropped_counter() { - struct FailingLogger; - impl CustomLogger for FailingLogger { - fn async_log_success_event<'a>( - &'a self, - _model_call_details: &'a ModelCallDetails, - _response_obj: &'a CallbackValue, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async { Err(LogError::channel_full()) }) - } - - fn async_log_failure_event<'a>( - &'a self, - _model_call_details: &'a ModelCallDetails, - _response_obj: Option<&'a CallbackValue>, - _timing: CallbackTiming, - ) -> LogFuture<'a> { - Box::pin(async { Err(LogError::channel_closed()) }) - } - } - let callbacks: Vec> = vec![Arc::new(FailingLogger)]; - let mut streaming = RealTimeStreaming::new( - callbacks, - "call_1".to_string(), - "gpt-realtime".to_string(), - RequestMetadata::default(), - ); - streaming.log_messages(SessionStatus::Success).await; - assert_eq!(streaming.dropped(), 1); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/AGENTS.md b/litellm-rust/crates/ai-gateway/src/routes/AGENTS.md deleted file mode 100644 index c675916f71a..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/AGENTS.md +++ /dev/null @@ -1,43 +0,0 @@ -# routes/ — the route template - -Every route follows the **same shape** so the layout is predictable. The rule: - -> **Each route module exposes `pub fn router() -> Router`.** -> `routes/mod.rs::app` merges them all and applies state once. Adding a route is: -> create the module, then add one `.merge(::router())` line. - -## Default: one file -A route is a single file containing `router()` + its handler(s) (handlers stay -private). This is the norm — don't split until it hurts. -``` -pub fn router() -> Router { Router::new().route(PATH, get(handle)) } -async fn handle(...) -> impl IntoResponse { ... } -``` -`health.rs` is the example. - -## Split out `service` when there's real logic -When a route has business logic worth testing without axum, put it in a sibling -`service` (a file, or a folder if the route grows). The route file stays the -**axum surface** (router + handler + any socket/SSE adapter); `service` is plain -Rust with **no axum types**, and its job is to pick the deployment and call the -`core` route entrypoint (see `messages/service.rs` calling -`litellm_core::messages::messages`). Never build a provider request, resolve a -key, or perform the provider call here. `realtime/` is the older example: -``` -realtime/ - mod.rs # axum surface: router() + handler + the WS<->events adapter - service.rs # pure logic: select deployment + call provider (no axum) — testable -``` -Split `service` further (or add `transport`, `repo`, …) only once a single file -genuinely gets hard to read. - -## Invariants -- **Auth is an extractor, not a manual call.** A handler requires auth by adding - `crate::auth::RequireMasterKey` to its arguments; it runs during extraction. - Never re-implement the check per route. -- **Handlers contain no business logic; `service` contains no axum types.** -- **No provider handlers in this crate.** Transforms, auth headers, and the - provider HTTP call live in `core/src//`. -- A route owns its paths in its own `router()`; `mod.rs` only merges. -- Cross-cutting concerns (logging, CORS, timeouts) → Tower layers in `mod.rs`, - not duplicated in handlers. diff --git a/litellm-rust/crates/ai-gateway/src/routes/health.rs b/litellm-rust/crates/ai-gateway/src/routes/health.rs deleted file mode 100644 index c64ca3a7199..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/health.rs +++ /dev/null @@ -1,24 +0,0 @@ -//! Health probes. Simple-route template: a `router()` plus its handlers, in one file. - -use axum::Router; -use axum::http::StatusCode; -use axum::routing::get; - -use crate::state::AppState; - -/// This route's contribution to the app router. -pub fn router() -> Router { - Router::new() - .route("/health/liveness", get(liveness)) - .route("/health/readiness", get(readiness)) -} - -/// The process is up. -async fn liveness() -> StatusCode { - StatusCode::OK -} - -/// The server is ready to accept traffic. -async fn readiness() -> StatusCode { - StatusCode::OK -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs deleted file mode 100644 index 3334053a0a4..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/mod.rs +++ /dev/null @@ -1,532 +0,0 @@ -//! `POST /v1/messages`, the Anthropic Messages HTTP surface. - -mod service; - -use axum::Router; -use axum::body::Body; -use axum::extract::{Json, State}; -use axum::http::StatusCode; -use axum::http::header::{CACHE_CONTROL, CONTENT_TYPE, HeaderMap, HeaderValue}; -use axum::response::{IntoResponse, Response}; -use axum::routing::post; -use litellm_core::Error; -use serde_json::{Map, Value}; - -use crate::auth::RequireMasterKey; -use crate::constants::{MESSAGES_HEADERS_NOT_FORWARDED, MESSAGES_ROUTE_PATH}; -use crate::state::AppState; - -/// This route's contribution to the app router. -pub fn router() -> Router { - Router::new().route(MESSAGES_ROUTE_PATH, post(handle)) -} - -#[tracing::instrument( - name = "messages_gateway_route", - target = "litellm::function_trace", - level = "trace", - skip_all -)] -async fn handle( - _auth: RequireMasterKey, - State(state): State, - headers: HeaderMap, - Json(body): Json, -) -> Result { - let extra_headers = forwarded_headers(&headers)?; - match service::run(&state.router, body, extra_headers) - .await - .map_err(MessagesRouteError::from)? - { - service::MessagesResponse::Json(body) => Ok(Json(body).into_response()), - service::MessagesResponse::Stream(upstream) => stream_response(upstream), - } -} - -fn stream_response(upstream: reqwest::Response) -> Result { - let content_type = upstream - .headers() - .get(CONTENT_TYPE) - .cloned() - .unwrap_or_else(|| HeaderValue::from_static("text/event-stream")); - let mut response = Response::builder() - .status( - StatusCode::from_u16(upstream.status().as_u16()).map_err(|error| { - MessagesRouteError(Error::InvalidResponse(format!( - "invalid upstream response status: {error}" - ))) - })?, - ) - .header(CONTENT_TYPE, content_type); - if let Some(value) = upstream.headers().get(CACHE_CONTROL) { - response = response.header(CACHE_CONTROL, value); - } - response - .body(Body::from_stream(upstream.bytes_stream())) - .map_err(|error| { - MessagesRouteError(Error::InvalidResponse(format!( - "failed to build streaming response: {error}" - ))) - }) -} - -fn forwarded_headers(headers: &HeaderMap) -> Result>, Error> { - let forwarded = headers - .iter() - .filter(|(name, _)| { - !MESSAGES_HEADERS_NOT_FORWARDED - .iter() - .any(|excluded| name.as_str().eq_ignore_ascii_case(excluded)) - }) - .map(|(name, value)| { - let value = value.to_str().map_err(|_| { - Error::InvalidRequest(format!("invalid value for header {}", name.as_str())) - })?; - Ok((name.to_string(), Value::String(value.to_string()))) - }) - .collect::, Error>>()?; - Ok((!forwarded.is_empty()).then_some(forwarded)) -} - -#[derive(Debug)] -struct MessagesRouteError(Error); - -impl From for MessagesRouteError { - fn from(error: Error) -> Self { - Self(error) - } -} - -impl IntoResponse for MessagesRouteError { - fn into_response(self) -> Response { - let (status, message) = match self.0 { - Error::InvalidRequest(message) => (StatusCode::BAD_REQUEST, message), - Error::InvalidProvider(_) | Error::Routing(_) => ( - StatusCode::NOT_FOUND, - "no messages deployment is configured for this model".to_string(), - ), - Error::Auth(_) - | Error::MissingApiKey { .. } - | Error::MissingAzureAiCredentials - | Error::MissingAzureDocumentIntelligenceCredentials - | Error::MissingReductoApiKey => ( - StatusCode::BAD_GATEWAY, - "messages provider authentication failed".to_string(), - ), - Error::Http { .. } - | Error::Network(_) - | Error::Connect(_) - | Error::InvalidResponse(_) - | Error::InvalidType { .. } - | Error::MissingField(_) - | Error::MissingDocumentUrl => ( - StatusCode::BAD_GATEWAY, - "messages provider request failed".to_string(), - ), - // The gateway has no Python implementation to decline to, so a - // request the core cannot serve is reported to the caller. The - // reason is a fixed internal string, never provider content. - Error::Unsupported(reason) => ( - StatusCode::BAD_REQUEST, - format!("messages request is not supported: {reason}"), - ), - }; - ( - status, - Json(serde_json::json!({"error": {"message": message}})), - ) - .into_response() - } -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use axum::body::Body; - use axum::http::Request; - use axum::http::StatusCode; - use axum::http::header::{CACHE_CONTROL, CONTENT_TYPE}; - use litellm_core::router::{Deployment, LiteLLMParams, Router as ModelRouter}; - use serde_json::json; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - use tokio::net::TcpListener; - use tower::ServiceExt; - - use super::super::app; - use crate::io::realtime_pool::RealtimePool; - use crate::state::AppState; - - fn state(model: &str, api_base: String, master_key: Option<&str>) -> AppState { - state_with_provider(model, model, api_base, master_key) - } - - fn state_with_provider( - model_alias: &str, - provider_model: &str, - api_base: String, - master_key: Option<&str>, - ) -> AppState { - AppState { - router: Arc::new(ModelRouter::new(vec![Deployment { - model_name: model_alias.to_string(), - litellm_params: LiteLLMParams { - model: format!("anthropic/{provider_model}"), - api_key: Some("upstream-key".to_string()), - api_base: Some(api_base), - }, - }])), - master_key: master_key.map(Arc::from), - loggers: Arc::new(Vec::new()), - realtime_pool: RealtimePool::disabled(), - } - } - - async fn upstream(listener: TcpListener) -> (String, tokio::task::JoinHandle) { - let address = listener.local_addr().expect("listener has address"); - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.expect("accepts request"); - let mut request = Vec::new(); - let mut buffer = [0_u8; 4096]; - loop { - let read = socket.read(&mut buffer).await.expect("reads request"); - request.extend_from_slice(&buffer[..read]); - if request.windows(4).any(|window| window == b"\r\n\r\n") { - break; - } - } - let request = String::from_utf8(request).expect("request is utf8"); - let content_length = request - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case("content-length") - .then(|| value.trim().parse::().ok()) - .flatten() - }) - .unwrap_or(0); - let header_end = request.find("\r\n\r\n").expect("request has headers") + 4; - let mut full_request = request.into_bytes(); - while full_request.len().saturating_sub(header_end) < content_length { - let read = socket.read(&mut buffer).await.expect("reads body"); - full_request.extend_from_slice(&buffer[..read]); - } - let request = String::from_utf8(full_request).expect("request is utf8"); - let body = r#"{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-test"}"#; - let response = format!( - "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", - body.len(), - body - ); - socket - .write_all(response.as_bytes()) - .await - .expect("writes response"); - request - }); - (format!("http://{address}"), server) - } - - async fn streaming_upstream( - listener: TcpListener, - status: u16, - content_type: &'static str, - body: &'static str, - ) -> (String, tokio::task::JoinHandle) { - let address = listener.local_addr().expect("listener has address"); - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.expect("accepts request"); - let mut request = Vec::new(); - let mut buffer = [0_u8; 4096]; - loop { - let read = socket.read(&mut buffer).await.expect("reads request"); - request.extend_from_slice(&buffer[..read]); - if request.windows(4).any(|window| window == b"\r\n\r\n") { - break; - } - } - let request_text = String::from_utf8(request).expect("request is utf8"); - let content_length = request_text - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case("content-length") - .then(|| value.trim().parse::().ok()) - .flatten() - }) - .unwrap_or(0); - let header_end = request_text.find("\r\n\r\n").expect("request has headers") + 4; - let mut full_request = request_text.into_bytes(); - while full_request.len().saturating_sub(header_end) < content_length { - let read = socket.read(&mut buffer).await.expect("reads body"); - full_request.extend_from_slice(&buffer[..read]); - } - let response = format!( - "HTTP/1.1 {status} OK\r\ncontent-type: {content_type}\r\ncache-control: no-cache\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", - body.len() - ); - socket - .write_all(response.as_bytes()) - .await - .expect("writes response"); - String::from_utf8(full_request).expect("request is utf8") - }); - (format!("http://{address}"), server) - } - - #[tokio::test] - async fn route_constructs_anthropic_upstream_request() { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); - let (api_base, server) = upstream(listener).await; - let app = app(state("claude-test", api_base, Some("master-key"))); - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/messages") - .header("authorization", "Bearer master-key") - .header("x-api-key", "request-upstream-key") - .header("anthropic-beta", "beta-feature") - .header("content-type", "application/json") - .body(Body::from( - json!({ - "model": "claude-test", - "max_tokens": 16, - "messages": [{"role": "user", "content": "hello"}] - }) - .to_string(), - )) - .expect("request builds"), - ) - .await - .expect("route responds"); - assert_eq!(response.status(), StatusCode::OK); - let body = axum::body::to_bytes(response.into_body(), usize::MAX) - .await - .expect("response body reads"); - assert_eq!( - serde_json::from_slice::(&body).expect("json")["id"], - "msg_1" - ); - let upstream_request = server.await.expect("upstream task completes"); - let (head, body) = upstream_request - .split_once("\r\n\r\n") - .expect("upstream request has body"); - let head = head.to_ascii_lowercase(); - assert!(head.contains("x-api-key: request-upstream-key")); - assert!(head.contains("anthropic-beta: beta-feature")); - assert!(!head.contains("authorization: bearer master-key")); - let body: serde_json::Value = serde_json::from_str(body).expect("upstream body is json"); - assert_eq!(body["model"], "claude-test"); - assert_eq!(body["messages"][0]["content"], "hello"); - } - - #[tokio::test] - async fn route_substitutes_model_alias_with_provider_model_upstream() { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); - let (api_base, server) = upstream(listener).await; - let app = app(state_with_provider( - "production", - "claude-sonnet-4-5", - api_base, - Some("master-key"), - )); - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/messages") - .header("authorization", "Bearer master-key") - .header("content-type", "application/json") - .body(Body::from( - json!({ - "model": "production", - "max_tokens": 16, - "messages": [{"role": "user", "content": "hello"}] - }) - .to_string(), - )) - .expect("request builds"), - ) - .await - .expect("route responds"); - assert_eq!(response.status(), StatusCode::OK); - let upstream_request = server.await.expect("upstream task completes"); - let (_, upstream_body) = upstream_request - .split_once("\r\n\r\n") - .expect("upstream request has body"); - let upstream_body: serde_json::Value = - serde_json::from_str(upstream_body).expect("upstream body is json"); - assert_eq!(upstream_body["model"], "claude-sonnet-4-5"); - assert_ne!(upstream_body["model"], "production"); - } - - #[tokio::test] - async fn route_streams_anthropic_events_without_buffering_or_reordering() { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); - let events = "event: message_start\ndata: {\"type\":\"message_start\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\"}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"; - let (api_base, server) = - streaming_upstream(listener, 200, "text/event-stream", events).await; - let app = app(state("claude-test", api_base, Some("master-key"))); - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/messages") - .header("authorization", "Bearer master-key") - .header("content-type", "application/json") - .body(Body::from( - json!({ - "model": "claude-test", - "max_tokens": 16, - "stream": true, - "messages": [{"role": "user", "content": "hello"}] - }) - .to_string(), - )) - .expect("request builds"), - ) - .await - .expect("route responds"); - assert_eq!(response.status(), StatusCode::OK); - assert_eq!( - response - .headers() - .get(CONTENT_TYPE) - .unwrap() - .to_str() - .unwrap(), - "text/event-stream" - ); - assert_eq!( - response - .headers() - .get(CACHE_CONTROL) - .unwrap() - .to_str() - .unwrap(), - "no-cache" - ); - let response_body = axum::body::to_bytes(response.into_body(), usize::MAX) - .await - .expect("response body reads"); - assert_eq!(response_body, events.as_bytes()); - let upstream_request = server.await.expect("upstream task completes"); - let (_, upstream_body) = upstream_request - .split_once("\r\n\r\n") - .expect("upstream request has body"); - assert_eq!( - serde_json::from_str::(upstream_body) - .expect("upstream body is json")["stream"], - true - ); - } - - #[tokio::test] - async fn route_maps_streaming_upstream_errors_before_starting_response() { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); - let (api_base, server) = streaming_upstream( - listener, - 429, - "application/json", - r#"{"error":"rate limited"}"#, - ) - .await; - let app = app(state("claude-test", api_base, Some("master-key"))); - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/messages") - .header("authorization", "Bearer master-key") - .header("content-type", "application/json") - .body(Body::from( - json!({ - "model": "claude-test", - "max_tokens": 16, - "stream": true, - "messages": [{"role": "user", "content": "hello"}] - }) - .to_string(), - )) - .expect("request builds"), - ) - .await - .expect("route responds"); - assert_eq!(response.status(), StatusCode::BAD_GATEWAY); - let response_body = axum::body::to_bytes(response.into_body(), usize::MAX) - .await - .expect("response body reads"); - assert_eq!( - serde_json::from_slice::(&response_body).expect("error is json")["error"] - ["message"], - "messages provider request failed" - ); - server.await.expect("upstream task completes"); - } - - #[tokio::test] - async fn route_rejects_missing_master_key() { - let app = app(state( - "claude-test", - "http://127.0.0.1:1".to_string(), - Some("master-key"), - )); - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/messages") - .header("content-type", "application/json") - .body(Body::from("{}")) - .expect("request builds"), - ) - .await - .expect("route responds"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - } - - #[tokio::test] - async fn route_rejects_invalid_master_key() { - let app = app(state( - "claude-test", - "http://127.0.0.1:1".to_string(), - Some("master-key"), - )); - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/messages") - .header("authorization", "Bearer wrong-key") - .header("content-type", "application/json") - .body(Body::from("{}")) - .expect("request builds"), - ) - .await - .expect("route responds"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - } - - #[tokio::test] - async fn route_rejects_malformed_json_without_panicking() { - let app = app(state( - "claude-test", - "http://127.0.0.1:1".to_string(), - Some("master-key"), - )); - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/messages") - .header("authorization", "Bearer master-key") - .header("content-type", "application/json") - .body(Body::from("{not-json")) - .expect("request builds"), - ) - .await - .expect("route responds"); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs b/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs deleted file mode 100644 index 5434719987b..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/messages/service.rs +++ /dev/null @@ -1,71 +0,0 @@ -use std::sync::Arc; - -use litellm_core::Error; -use litellm_core::constants::ANTHROPIC_MESSAGES_PROVIDER; -use litellm_core::messages::types::MessagesRequest; -use litellm_core::messages::{messages, messages_stream}; -use litellm_core::router::Router; -use serde_json::{Map, Value}; - -pub(crate) enum MessagesResponse { - Json(Value), - Stream(reqwest::Response), -} - -#[tracing::instrument( - name = "messages_gateway_service", - target = "litellm::function_trace", - level = "trace", - skip_all -)] -pub async fn run( - router: &Arc, - body: Value, - extra_headers: Option>, -) -> Result { - let model = body - .get("model") - .and_then(Value::as_str) - .map(str::trim) - .filter(|model| !model.is_empty()) - .ok_or_else(|| Error::InvalidRequest("messages body requires a model".to_string()))?; - let deployment = router - .get_available_deployment(model) - .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; - let provider_model = deployment.litellm_params.model.as_str(); - let upstream_model = provider_model - .split_once('/') - .map_or(provider_model, |(_, model)| model); - let custom_llm_provider = if provider_model.contains('/') { - None - } else { - Some(ANTHROPIC_MESSAGES_PROVIDER) - }; - let mut body = body; - body.as_object_mut() - .ok_or_else(|| Error::InvalidRequest("messages body must be an object".to_string()))? - .insert( - "model".to_string(), - Value::String(upstream_model.to_string()), - ); - - let request = MessagesRequest { - model: provider_model, - body, - api_key: deployment.litellm_params.api_key.as_deref(), - api_base: deployment.litellm_params.api_base.as_deref(), - custom_llm_provider, - extra_headers, - timeout: None, - }; - if request.body.get("stream").and_then(Value::as_bool) == Some(true) { - return messages_stream(request).await.map(MessagesResponse::Stream); - } - - let response = messages(request).await?; - serde_json::to_value(response) - .map(MessagesResponse::Json) - .map_err(|err| { - Error::InvalidResponse(format!("failed to serialize messages response: {err}")) - }) -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/mod.rs deleted file mode 100644 index 71b05c7d64b..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! HTTP routes. -//! -//! **Template:** every route module exposes `pub fn router() -> Router` -//! that mounts its own paths; [`app`] merges them. A trivial route is a single -//! file (`health.rs`); a non-trivial one is a folder (`realtime/`) with -//! `handler` (entry) + `service` (logic) + `transport` (adapters). See AGENTS.md. - -pub mod health; -pub mod messages; -pub mod realtime; -pub mod responses; - -use axum::Router; - -use crate::state::AppState; - -/// Assemble the application router by merging every route module's `router()`. -pub fn app(state: AppState) -> Router { - Router::new() - .merge(health::router()) - .merge(messages::router()) - .merge(realtime::router()) - .merge(responses::router()) - .with_state(state) -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/realtime/README.md b/litellm-rust/crates/ai-gateway/src/routes/realtime/README.md deleted file mode 100644 index 3301576bb85..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/realtime/README.md +++ /dev/null @@ -1,87 +0,0 @@ -# Realtime route (`GET /v1/realtime`) - -Proxies OpenAI's realtime WebSocket. `mod.rs` is the axum surface (handler + -socket↔events adapter); `service.rs` is the pure logic (select a deployment, then -splice client ↔ upstream). The pool itself lives in -`crates/providers/src/realtime_pool.rs`. - -## Connection pooling - -### The problem - -The gateway's realtime overhead lives **entirely in session establishment**. On each -client connect it dials a *fresh* upstream WS to OpenAI and waits for -`session.created` before it can serve. Measured at 5000 calls / 500 concurrency, the -fresh-dial session phase is **~360 ms** vs **~7 ms** direct; dial, first-audio, and -streaming add ~0. So the one lever is removing that per-connect handshake from the -critical path. - -### The idea - -Keep a few upstream OpenAI sockets **already connected and already past -`session.created`** (buffered). On a client connect, hand off a warm socket — relay -its buffered `session.created` instantly (a local `Vec::pop`, sub-millisecond) and -splice exactly as a fresh dial would. A background task keeps the pool topped up. On -a miss or dead socket we fall back to fresh-dial: the pool is a latency optimization, -never a correctness dependency. - -``` - ┌───────────────────────────────────────┐ - client connect ──────► │ routes/realtime → service::run │ - │ pool.take(key) │ - │ hit → relay buffered │ - │ session.created, then splice │ - │ miss → fresh dial (original path) │ - └───────────────┬───────────────────────┘ - │ replenish (async, concurrent) - ┌───────────────▼───────────────────────┐ - background task ─────► │ RealtimePool: per-key warm sockets │ - │ each = { ws, buffered session.created}│ - │ liveness-checked before handoff │ - └─────────────────────────────────────────┘ -``` - -A warm session is indistinguishable from a fresh one: OpenAI sends `session.created` -unprompted on connect, we pre-read exactly that one frame and relay it on handoff, -and we send nothing else on the socket before a client exists — so the client's first -`session.update` behaves identically either way. - -### Sizing - -Each warm socket serves **exactly one** session (realtime isn't multiplexed), so the -pool is sized to the **peak concurrent connects per instance**, not total live -connections: - -``` -REALTIME_POOL_SIZE ≈ peak_concurrency / instance_count -``` - -e.g. 500 concurrency over 10 instances → ~50–64 per instance. The replenisher dials -the missing sockets **concurrently**, so a drained pool refills in ~one handshake -window and keeps supply close to the connect rate. Over-provisioning just burns idle -upstream sockets, which is why warm sockets are short-lived -(`REALTIME_POOL_MAX_IDLE_SECS`). - -### Config - -| env | default | meaning | -| ----------------------------- | ------- | --------------------------------------------------------------- | -| `REALTIME_POOL_SIZE` | `4` | target warm sockets per key. `0` disables pooling (fresh-dial). | -| `REALTIME_POOL_MAX_IDLE_SECS` | `30` | max time a warm socket sits before it's closed and replaced. | - -### Notes - -- **Miss / dead socket → fresh dial.** Burst beyond warm supply, or a socket that - died, never blocks or fails — it falls back to the original path. The pool can only - make a connect faster, never slower or more fragile. -- **Auth scope.** The pool key includes `api_key`, so a warm socket is only handed to - a request resolving to the same key — no cross-tenant reuse. -- **Idle billing.** Warm sockets are liveness-checked at handoff and capped at - `REALTIME_POOL_MAX_IDLE_SECS` to bound idle billing and dodge OpenAI's idle timeout. -- **Replenish backoff.** If a key's warm-up dials all fail (invalid credentials, an - unreachable upstream), the replenisher puts that key into exponential backoff - (500 ms → 30 s cap) instead of re-dialing it every tick. This bounds connection - attempts against a broken key so it can't exhaust upstream rate limits and degrade - valid cold-path traffic; the backoff resets the moment a dial succeeds. - -Benchmarks and repro: `../../benchmarks/realtime/README.md`. diff --git a/litellm-rust/crates/ai-gateway/src/routes/realtime/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/realtime/mod.rs deleted file mode 100644 index f9144ad1fdb..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/realtime/mod.rs +++ /dev/null @@ -1,166 +0,0 @@ -//! `GET /v1/realtime` (WebSocket). -//! -//! This file is the **axum surface**: `router()`, the handler, and the small -//! socket↔events adapter. The pure logic (no axum) lives in [`service`]. Auth is -//! the `RequireMasterKey` extractor, so the handler stays thin. - -mod service; - -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{SystemTime, UNIX_EPOCH}; - -use crate::io::realtime_pool::RealtimePool; -use axum::Router; -use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; -use axum::extract::{Query, State}; -use axum::http::StatusCode; -use axum::response::Response; -use axum::routing::get; -use futures_util::{SinkExt, StreamExt}; -use litellm_core::realtime::types::RealtimeEvent; -use litellm_core::router::Router as ModelRouter; -use serde::Deserialize; - -use crate::auth::RequireMasterKey; -use crate::integrations::custom_logger::CustomLogger; -use crate::integrations::types::RequestMetadata; -use crate::realtime::streaming::{RealTimeStreaming, SessionStatus}; -use crate::state::AppState; - -/// Process-local monotonic counter, mixed into the per-session call id so two -/// sessions opened in the same nanosecond still get distinct ids. -static CALL_SEQ: AtomicU64 = AtomicU64::new(0); - -/// Generate a per-connection `litellm_call_id`. No external uuid dep: epoch -/// nanos + a process-local sequence is unique enough for log correlation. -fn new_call_id() -> String { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); - let seq = CALL_SEQ.fetch_add(1, Ordering::Relaxed); - format!("rt-{nanos:x}-{seq:x}") -} - -/// This route's contribution to the app router. -pub fn router() -> Router { - Router::new().route("/v1/realtime", get(handle)) -} - -#[derive(Debug, Deserialize)] -struct RealtimeQuery { - model: String, -} - -/// Auth runs via the `RequireMasterKey` extractor. We validate the model BEFORE -/// the upgrade so failures are clean HTTP (400/404), not a socket that opens then -/// closes, then hand the socket to `bridge`. -async fn handle( - _auth: RequireMasterKey, - ws: WebSocketUpgrade, - State(state): State, - Query(query): Query, -) -> Result { - if query.model.trim().is_empty() { - return Err(( - StatusCode::BAD_REQUEST, - "missing 'model' query param".to_string(), - )); - } - if !state.router.has_deployment(&query.model) { - return Err(( - StatusCode::NOT_FOUND, - format!("no deployment for model '{}'", query.model), - )); - } - - let router = state.router.clone(); - let pool = state.realtime_pool.clone(); - let loggers = state.loggers.clone(); - let master_key = state.master_key.clone(); - let model = query.model; - Ok(ws.on_upgrade(move |socket| bridge(socket, router, pool, loggers, master_key, model))) -} - -/// Adapt the axum socket (text frames) to the typed-event `Stream`/`Sink` the -/// service wants, keeping axum types out of `service`. -/// -/// This is also the realtime-logging seam: every upstream→client event (the -/// direction carrying `session.created` and `response.done` with usage) is fed -/// to a [`RealTimeStreaming`] collector via the splice's `observe` callback. The -/// observe is O(1) and never buffers frames. When the splice returns (any of the -/// three break paths — client disconnect, upstream close, idle timeout), we flush -/// one logging payload to the registered callbacks. -async fn bridge( - socket: WebSocket, - router: Arc, - pool: Arc, - loggers: Arc>>, - master_key: Option>, - model: String, -) { - let (ws_sink, ws_stream) = socket.split(); - - // Attribute the spend log to the key that authenticated this session (the - // master key — the gateway is master-key auth). A non-null user_api_key_hash - // is required for the Python spend logger to write a SpendLogs row. - // - // SECURITY: hash the key — never send the raw credential. This field fans out - // to spend logs and every callback integration; the SHA-256 (matching the - // proxy's hash_token) keeps the plaintext master key out of all of them while - // still matching the key's hash in LiteLLM_SpendLogs. - let metadata = RequestMetadata { - user_api_key_hash: master_key.as_deref().map(crate::auth::hash_token), - ..RequestMetadata::default() - }; - - // Owned by THIS task only. The splice observes it via a synchronous `&mut` - // callback (below), so there is no Arc/Mutex/atomic on the per-frame hot - // path — just a monomorphized FnMut mutating stack-local fields. This is - // what lets observe scale: 10K concurrent sessions = 10K independent - // collectors, zero cross-task synchronization. - let mut collector = RealTimeStreaming::new( - loggers.as_ref().clone(), - new_call_id(), - model.clone(), - metadata, - ); - - let client_in = ws_stream.filter_map(|message| async move { - match message { - Ok(Message::Text(text)) => serde_json::from_str::(&text).ok(), - _ => None, - } - }); - // Plain forwarding sink — no observe here anymore. - let client_out = ws_sink.with(|event: RealtimeEvent| async move { - Ok::(Message::Text( - serde_json::to_string(&event).unwrap_or_default(), - )) - }); - - futures_util::pin_mut!(client_in, client_out); - - // The observe closure borrows `&mut collector` for the duration of the - // splice; the borrow ends when `run` returns, freeing the collector for the - // single post-session `log_messages` flush. `run` picks a pooled (warm) or - // fresh upstream — observe fires on the upstream arm either way. - let result = service::run( - &router, - &pool, - &model, - None, - |event: &RealtimeEvent| collector.observe(event), - client_in, - client_out, - ) - .await; - - let status = if result.is_ok() { - SessionStatus::Success - } else { - SessionStatus::Failure - }; - collector.log_messages(status).await; -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs b/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs deleted file mode 100644 index f7bbb37dff4..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/realtime/service.rs +++ /dev/null @@ -1,77 +0,0 @@ -//! Business logic: select a deployment with the (pure) core router, then call the -//! provider splice. The seam between `core::router` (selection only) and -//! `io` (the actual WebSocket I/O). -//! -//! On connect we try a pre-warmed upstream from the pool (handshake already paid, -//! `session.created` buffered) and relay it instantly. On a pool miss or dead warm -//! socket we fresh-dial exactly as before — the pool is never on the critical path -//! for correctness, only latency. - -use std::time::Duration; - -use crate::io::realtime_pool::{RealtimePool, upstream_key}; -use futures_util::{Sink, Stream}; -use litellm_core::error::Error; -use litellm_core::realtime::types::RealtimeEvent; -use litellm_core::router::Router; - -/// Select a deployment for `model` and splice the client stream to the provider. -/// -/// `pool` supplies a pre-warmed upstream when one is available; otherwise we -/// fresh-dial. A disabled pool always misses, so this collapses to the original -/// fresh-dial behavior. -pub async fn run( - router: &Router, - pool: &RealtimePool, - model: &str, - idle_timeout: Option, - observe: impl FnMut(&RealtimeEvent) + Send, - client_in: In, - client_out: Out, -) -> Result<(), Error> -where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - >::Error: std::fmt::Display, -{ - let deployment = router - .get_available_deployment(model) - .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; - let params = &deployment.litellm_params; - // Strip a leading `openai/` so the OpenAI-only realtime fn gets the bare model. - let provider_model = params - .model - .strip_prefix("openai/") - .unwrap_or(¶ms.model); - - // Warm path: take a pooled upstream (handshake already paid) and relay its - // buffered session.created immediately. On miss/dead socket fall through. - if let Some(key) = upstream_key( - provider_model, - params.api_key.as_deref(), - params.api_base.as_deref(), - ) && let Some(handoff) = pool.take(&key) - { - return crate::io::realtime::realtime_warm( - provider_model, - handoff, - idle_timeout, - observe, - client_in, - client_out, - ) - .await; - } - - // Cold path: fresh dial (the original behavior). - crate::io::realtime::realtime( - provider_model, - params.api_key.as_deref(), - params.api_base.as_deref(), - idle_timeout, - observe, - client_in, - client_out, - ) - .await -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/responses/mod.rs b/litellm-rust/crates/ai-gateway/src/routes/responses/mod.rs deleted file mode 100644 index a94853e106d..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/responses/mod.rs +++ /dev/null @@ -1,348 +0,0 @@ -mod service; - -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{SystemTime, UNIX_EPOCH}; - -use axum::Router; -use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; -use axum::extract::{Query, State}; -use axum::http::StatusCode; -use axum::response::Response; -use axum::routing::get; -use futures_util::{Sink, SinkExt, StreamExt}; -use litellm_core::responses::types::{ResponsesErrorFrame, ResponsesWsEvent, ResponsesWsEventType}; -use litellm_core::router::Router as ModelRouter; -use serde::Deserialize; - -use crate::auth::RequireMasterKey; -use crate::integrations::custom_logger::CustomLogger; -use crate::integrations::types::RequestMetadata; -use crate::state::AppState; - -static CALL_SEQ: AtomicU64 = AtomicU64::new(0); - -fn new_call_id() -> String { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_nanos()) - .unwrap_or(0); - let sequence = CALL_SEQ.fetch_add(1, Ordering::Relaxed); - format!("respws-{nanos:x}-{sequence:x}") -} - -pub fn router() -> Router { - Router::new() - .route("/v1/responses", get(handle)) - .route("/responses", get(handle)) -} - -#[derive(Debug, Deserialize)] -struct ResponsesQuery { - model: Option, -} - -async fn handle( - _auth: RequireMasterKey, - ws: WebSocketUpgrade, - State(state): State, - Query(query): Query, -) -> Result { - if let Some(model) = query.model.as_deref() { - validate_model(&state.router, model)?; - } - let router = state.router.clone(); - let loggers = state.loggers.clone(); - let master_key = state.master_key.clone(); - Ok(ws.on_upgrade(move |socket| bridge(socket, router, loggers, master_key, query.model))) -} - -fn validate_model(router: &ModelRouter, model: &str) -> Result<(), (StatusCode, String)> { - if model.trim().is_empty() { - return Err(( - StatusCode::BAD_REQUEST, - "missing 'model' query param".to_string(), - )); - } - let Some(deployment) = router.get_available_deployment(model) else { - return Err(( - StatusCode::NOT_FOUND, - format!("no deployment for model '{model}'"), - )); - }; - if deployment.litellm_params.model.contains('/') - && !deployment.litellm_params.model.starts_with("openai/") - { - return Err(( - StatusCode::BAD_REQUEST, - "Responses WebSocket route supports OpenAI deployments only".to_string(), - )); - } - Ok(()) -} - -async fn send_error_and_close(sink: &mut S, message: String) -where - S: futures_util::Sink + Unpin, - S::Error: std::fmt::Display, -{ - if let Ok(payload) = serde_json::to_string(&ResponsesErrorFrame::invalid_request(message)) { - let _ = sink.send(Message::Text(payload)).await; - } - let _ = sink - .send(Message::Close(Some(axum::extract::ws::CloseFrame { - code: 1008, - reason: "Pre-call error".into(), - }))) - .await; - let _ = sink.close().await; -} - -struct ResponseClientSink { - sink: futures_util::stream::SplitSink, -} - -impl Sink for ResponseClientSink { - type Error = axum::Error; - - fn poll_ready( - mut self: std::pin::Pin<&mut Self>, - context: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - std::pin::Pin::new(&mut self.sink).poll_ready(context) - } - - fn start_send( - mut self: std::pin::Pin<&mut Self>, - item: ResponsesWsEvent, - ) -> Result<(), Self::Error> { - let payload = serde_json::to_string(&item).map_err(axum::Error::new)?; - std::pin::Pin::new(&mut self.sink).start_send(Message::Text(payload)) - } - - fn poll_flush( - mut self: std::pin::Pin<&mut Self>, - context: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - std::pin::Pin::new(&mut self.sink).poll_flush(context) - } - - fn poll_close( - mut self: std::pin::Pin<&mut Self>, - context: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - std::pin::Pin::new(&mut self.sink).poll_close(context) - } -} - -impl ResponseClientSink { - async fn close_with_code(&mut self, code: u16, reason: &'static str) { - let _ = self - .sink - .send(Message::Close(Some(axum::extract::ws::CloseFrame { - code, - reason: reason.into(), - }))) - .await; - let _ = self.sink.close().await; - } -} - -async fn bridge( - socket: WebSocket, - router: Arc, - loggers: Arc>>, - master_key: Option>, - requested_model: Option, -) { - let (mut ws_sink, ws_stream) = socket.split(); - let (model, first_frame, stream) = if let Some(model) = requested_model { - (model, None, ws_stream) - } else { - let mut stream = ws_stream; - let first = match stream.next().await { - Some(Ok(Message::Text(text))) => { - match serde_json::from_str::(&text) { - Ok(event) => event, - Err(_) => { - send_error_and_close( - &mut ws_sink, - "Invalid JSON in response.create event".to_string(), - ) - .await; - return; - } - } - } - _ => { - send_error_and_close(&mut ws_sink, "Missing response.create event".to_string()) - .await; - return; - } - }; - let Some(model) = first.model().filter(|value| !value.trim().is_empty()) else { - send_error_and_close( - &mut ws_sink, - "Missing model in response.create event".to_string(), - ) - .await; - return; - }; - if first.event_type != ResponsesWsEventType::ResponseCreate { - send_error_and_close( - &mut ws_sink, - "First frame must be a response.create event".to_string(), - ) - .await; - return; - } - (model.to_string(), Some(first), stream) - }; - if let Err((status, message)) = validate_model(&router, &model) { - let _ = status; - let _ = message; - send_error_and_close(&mut ws_sink, "Unknown model deployment".to_string()).await; - return; - } - - let call_id = new_call_id(); - let metadata = RequestMetadata { - user_api_key_hash: master_key.as_deref().map(crate::auth::hash_token), - ..RequestMetadata::default() - }; - let client_in = Box::pin(stream.filter_map(|message| async move { - match message { - Ok(Message::Text(text)) => serde_json::from_str::(&text).ok(), - _ => None, - } - })); - let mut client_out = ResponseClientSink { sink: ws_sink }; - let result = service::run( - &router, - &model, - first_frame, - None, - loggers, - call_id, - metadata, - client_in, - &mut client_out, - ) - .await; - if result.is_err() { - client_out - .close_with_code(1011, "Internal server error") - .await; - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::io::realtime_pool::RealtimePool; - use crate::state::AppState; - use axum::body::Body; - use axum::http::Request; - use litellm_core::router::Router as ModelRouter; - use serde_json::json; - use std::pin::Pin; - use std::sync::Arc; - use std::task::{Context, Poll}; - use tower::ServiceExt; - - struct RecordingSink { - messages: Vec, - } - - impl Sink for RecordingSink { - type Error = std::convert::Infallible; - - fn poll_ready( - self: Pin<&mut Self>, - _context: &mut Context<'_>, - ) -> Poll> { - Poll::Ready(Ok(())) - } - - fn start_send(mut self: Pin<&mut Self>, item: Message) -> Result<(), Self::Error> { - self.messages.push(item); - Ok(()) - } - - fn poll_flush( - self: Pin<&mut Self>, - _context: &mut Context<'_>, - ) -> Poll> { - Poll::Ready(Ok(())) - } - - fn poll_close( - self: Pin<&mut Self>, - _context: &mut Context<'_>, - ) -> Poll> { - Poll::Ready(Ok(())) - } - } - - #[tokio::test] - async fn pre_call_error_matches_python_frame_and_close() { - let mut sink = RecordingSink { - messages: Vec::new(), - }; - send_error_and_close(&mut sink, "missing model".to_string()).await; - let Message::Text(payload) = &sink.messages[0] else { - panic!("expected error text frame"); - }; - assert_eq!( - serde_json::from_str::(payload).expect("error json"), - json!({ - "type": "error", - "error": { - "type": "invalid_request_error", - "message": "missing model" - } - }) - ); - assert_eq!( - sink.messages[1], - Message::Close(Some(axum::extract::ws::CloseFrame { - code: 1008, - reason: "Pre-call error".into(), - })) - ); - } - - fn state() -> AppState { - AppState { - router: Arc::new(ModelRouter::default()), - master_key: Some(Arc::from("master-key")), - loggers: Arc::new(Vec::new()), - realtime_pool: RealtimePool::disabled(), - } - } - - #[tokio::test] - async fn auth_rejects_responses_upgrade_before_handler() { - let request = Request::builder() - .uri("/responses?model=known") - .body(Body::empty()) - .expect("request"); - let response = router() - .with_state(state()) - .oneshot(request) - .await - .expect("response"); - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - } - - #[test] - fn unknown_query_model_is_rejected_before_upgrade() { - assert_eq!( - validate_model(&ModelRouter::default(), "unknown").expect_err("unknown model"), - ( - StatusCode::NOT_FOUND, - "no deployment for model 'unknown'".to_string() - ) - ); - } -} diff --git a/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs b/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs deleted file mode 100644 index e8f840c0c8e..00000000000 --- a/litellm-rust/crates/ai-gateway/src/routes/responses/service.rs +++ /dev/null @@ -1,156 +0,0 @@ -use std::sync::Arc; -use std::time::Duration; - -use futures_util::{Sink, Stream}; -use litellm_core::Error; -use litellm_core::call_lifecycle::{CallLifecycle, CallLifecycleContext}; -use litellm_core::responses::instrumentation::{ - ResponsesWsCallbackPayload, ResponsesWsInstrumentation, ResponsesWsLogOutcome, - ResponsesWsMetadata, -}; -use litellm_core::responses::types::ResponsesWsEvent; - -use crate::integrations::custom_logger::{ - CallbackTiming, CallbackValue, CustomLogger, CustomLoggerRunner, LoggingError, ModelCallDetails, -}; -use crate::integrations::types::RequestMetadata; - -#[allow(clippy::too_many_arguments)] -pub async fn run( - router: &litellm_core::router::Router, - model: &str, - first_frame: Option, - idle_timeout: Option, - loggers: Arc>>, - call_id: String, - metadata: RequestMetadata, - client_in: In, - client_out: Out, -) -> Result<(), Error> -where - In: Stream + Unpin + Send, - Out: Sink + Unpin + Send, - Out::Error: std::fmt::Display, -{ - let deployment = router - .get_available_deployment(model) - .ok_or_else(|| Error::Routing(format!("no deployment available for model '{model}'")))?; - let params = &deployment.litellm_params; - let provider_model = params - .model - .strip_prefix("openai/") - .unwrap_or(¶ms.model); - if params.model.contains('/') && !params.model.starts_with("openai/") { - return Err(Error::InvalidProvider( - "Responses WebSocket route supports OpenAI deployments only".to_string(), - )); - } - let instrumentation = Arc::new(ResponsesWsInstrumentation::new( - call_id.clone(), - model, - ResponsesWsMetadata { - user_api_key_hash: metadata.user_api_key_hash, - user_api_key_user_id: metadata.user_api_key_user_id, - user_api_key_team_id: metadata.user_api_key_team_id, - }, - )); - let observer_instrumentation = Arc::clone(&instrumentation); - let context = CallLifecycleContext::new("responses_websocket", model, "openai", call_id); - let result = CallLifecycle::default() - .run(context, (), instrumentation.as_ref(), |_| async move { - crate::io::responses_ws::async_responses_websocket( - provider_model, - params.api_key.as_deref(), - params.api_base.as_deref(), - first_frame, - idle_timeout, - move |event| { - observer_instrumentation.observe(event); - }, - client_in, - client_out, - ) - .await - }) - .await; - let outcome = instrumentation.take_or_build_outcome(result.is_ok()); - dispatch_outcome(loggers, outcome).await; - result -} - -async fn dispatch_outcome( - loggers: Arc>>, - outcome: ResponsesWsLogOutcome, -) { - let runner = CustomLoggerRunner::new(loggers.as_ref().clone()); - match outcome { - ResponsesWsLogOutcome::Success { payload, callback } => { - let (details, response, start_time, end_time) = logging_values(payload, callback, None); - let _ = runner - .async_log_success_event( - &details, - &response, - CallbackTiming::new(start_time, end_time), - ) - .await; - } - ResponsesWsLogOutcome::Failure { - payload, - callback, - error_message, - error_kind, - } => { - let error = LoggingError { - message: error_message, - kind: error_kind, - }; - let (details, response, start_time, end_time) = - logging_values(payload, callback, Some(error)); - let _ = runner - .async_log_failure_event( - &details, - Some(&response), - CallbackTiming::new(start_time, end_time), - ) - .await; - } - } -} - -fn logging_values( - payload: litellm_core::responses::instrumentation::ResponsesWsLogPayload, - callback: ResponsesWsCallbackPayload, - error: Option, -) -> (ModelCallDetails, CallbackValue, f64, f64) { - let start_time = payload.start_time; - let end_time = payload.end_time; - let callback = CallbackValue::new(callback.object, callback.value); - let details = ModelCallDetails::from_standard_logging_payload( - crate::integrations::types::StandardLoggingPayload { - id: payload.id, - litellm_call_id: payload.litellm_call_id, - call_type: payload.call_type, - model: payload.model, - custom_llm_provider: payload.custom_llm_provider, - response_cost: payload.response_cost, - prompt_tokens: payload.usage.prompt_tokens, - completion_tokens: payload.usage.completion_tokens, - total_tokens: payload.usage.total_tokens, - start_time: payload.start_time, - end_time: payload.end_time, - stream: payload.stream, - metadata: crate::integrations::types::StandardLoggingMetadata { - user_api_key_hash: payload.metadata.user_api_key_hash, - user_api_key_user_id: payload.metadata.user_api_key_user_id, - user_api_key_team_id: payload.metadata.user_api_key_team_id, - ..Default::default() - }, - messages: None, - }, - ); - let details = match error { - Some(error) => details.with_failure_error(error), - None => details, - }; - (details, callback, start_time, end_time) -} diff --git a/litellm-rust/crates/ai-gateway/src/state.rs b/litellm-rust/crates/ai-gateway/src/state.rs deleted file mode 100644 index 3b61d8309ea..00000000000 --- a/litellm-rust/crates/ai-gateway/src/state.rs +++ /dev/null @@ -1,21 +0,0 @@ -use std::sync::Arc; - -use crate::io::realtime_pool::RealtimePool; -use litellm_core::router::Router; - -use crate::integrations::custom_logger::CustomLogger; - -/// Shared application state handed to every route handler. -#[derive(Clone)] -pub struct AppState { - pub router: Arc, - /// The gateway master key. Any caller presenting it as a bearer token may - /// invoke the gateway. `None` → auth not configured (routes fail closed). - pub master_key: Option>, - /// Logging callbacks fanned out at the end of each realtime session. - pub loggers: Arc>>, - /// Pre-warmed upstream realtime connection pool. Disabled - /// (`RealtimePool::disabled()`) when `REALTIME_POOL_SIZE=0`, in which case - /// every realtime connect fresh-dials exactly as before. - pub realtime_pool: Arc, -} diff --git a/litellm-rust/crates/ai-gateway/src/trace_parity.rs b/litellm-rust/crates/ai-gateway/src/trace_parity.rs deleted file mode 100644 index 7540a71fb12..00000000000 --- a/litellm-rust/crates/ai-gateway/src/trace_parity.rs +++ /dev/null @@ -1,100 +0,0 @@ -//! Harness-only in-process adapters. Never mounted as production routes. - -use std::sync::Arc; - -use axum::body::{Body, to_bytes}; -use axum::http::header::{AUTHORIZATION, CONTENT_TYPE}; -use axum::http::{Request, StatusCode}; -use litellm_core::Error; -use litellm_core::router::{Deployment, LiteLLMParams, Router as ModelRouter}; -use serde::Serialize; -use serde_json::Value; -use tower::ServiceExt; -use tracing::instrument::WithSubscriber; - -use crate::io::realtime_pool::RealtimePool; -use crate::routes; -use crate::state::AppState; - -#[derive(Debug, Serialize)] -pub struct GatewayResponse { - pub status: u16, - pub body: Value, -} - -#[derive(Debug, Serialize)] -pub struct TracedGatewayResponse { - pub response: Option, - pub error: Option, - pub trace: Vec, -} - -pub async fn traced_request( - path: String, - model_alias: String, - provider_model: String, - api_base: String, - body: Value, -) -> TracedGatewayResponse { - let trace = litellm_core::observability::FunctionTrace::default(); - let result = request(path, model_alias, provider_model, api_base, body) - .with_subscriber(trace.dispatcher()) - .await; - let events = trace.events(); - match result { - Ok(response) => TracedGatewayResponse { - response: Some(response), - error: None, - trace: events, - }, - Err(error) => TracedGatewayResponse { - response: None, - error: Some(error.to_string()), - trace: events, - }, - } -} - -pub async fn request( - path: String, - model_alias: String, - provider_model: String, - api_base: String, - body: Value, -) -> Result { - let state = AppState { - router: Arc::new(ModelRouter::new(vec![Deployment { - model_name: model_alias, - litellm_params: LiteLLMParams { - model: provider_model, - api_key: Some("trace-provider-key".to_string()), - api_base: Some(api_base), - }, - }])), - master_key: Some(Arc::from("trace-master-key")), - loggers: Arc::new(Vec::new()), - realtime_pool: RealtimePool::disabled(), - }; - let request = Request::builder() - .method("POST") - .uri(path) - .header(AUTHORIZATION, "Bearer trace-master-key") - .header(CONTENT_TYPE, "application/json") - .body(Body::from(body.to_string())) - .map_err(|error| Error::InvalidRequest(error.to_string()))?; - let response = match routes::app(state).oneshot(request).await { - Ok(response) => response, - Err(error) => match error {}, - }; - let status: StatusCode = response.status(); - let bytes = to_bytes(response.into_body(), usize::MAX) - .await - .map_err(|error| Error::InvalidResponse(error.to_string()))?; - let body = serde_json::from_slice(&bytes).map_err(|error| { - Error::InvalidResponse(format!("gateway returned invalid JSON: {error}")) - })?; - Ok(GatewayResponse { - status: status.as_u16(), - body, - }) -} diff --git a/litellm-rust/crates/ai-gateway/tests/crypto_provider_wiring.rs b/litellm-rust/crates/ai-gateway/tests/crypto_provider_wiring.rs deleted file mode 100644 index ac37440d682..00000000000 --- a/litellm-rust/crates/ai-gateway/tests/crypto_provider_wiring.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! Guards the wiring, not just the helper: a `wss://` dial through the public -//! API has to resolve its own crypto provider, in a test binary where nothing -//! has installed a process-wide one, and has to leave it uninstalled. - -use std::time::Duration; - -use futures_util::{sink, stream}; -use litellm_ai_gateway::io::responses_ws::async_responses_websocket; -use tokio::net::TcpListener; - -async fn dead_tls_server() -> u16 { - let listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("bind a loopback port"); - let port = listener - .local_addr() - .expect("read the bound address") - .port(); - - tokio::spawn(async move { - while let Ok((stream, _peer)) = listener.accept().await { - drop(stream); - } - }); - - port -} - -#[tokio::test] -async fn dialing_wss_returns_an_error_instead_of_panicking() { - let port = dead_tls_server().await; - - let result = async_responses_websocket( - "gpt-5", - Some("test-key"), - Some(&format!("wss://127.0.0.1:{port}/")), - None, - Some(Duration::from_secs(10)), - |_| {}, - stream::empty(), - sink::drain(), - ) - .await; - - assert!( - result.is_err(), - "a plain TCP server cannot finish a TLS handshake" - ); - assert!( - rustls::crypto::CryptoProvider::get_default().is_none(), - "the dial settles its provider on its own connector, not process-wide" - ); -} diff --git a/litellm-rust/crates/config/Cargo.toml b/litellm-rust/crates/config/Cargo.toml deleted file mode 100644 index ae9710266a3..00000000000 --- a/litellm-rust/crates/config/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "litellm-config" -version = "0.1.0" -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -litellm-core.workspace = true -pyo3 = { workspace = true, features = ["auto-initialize"], optional = true } -serde_json.workspace = true -thiserror.workspace = true - -[features] -default = [] -python = ["dep:pyo3"] diff --git a/litellm-rust/crates/config/src/error.rs b/litellm-rust/crates/config/src/error.rs deleted file mode 100644 index cec7bc5c110..00000000000 --- a/litellm-rust/crates/config/src/error.rs +++ /dev/null @@ -1,11 +0,0 @@ -use thiserror::Error as ThisError; - -#[derive(Debug, ThisError)] -pub enum Error { - #[error("read_model_list failed: {0}")] - PythonLoading(String), - #[error("serializing model_list failed: {0}")] - Serialization(String), - #[error("parsing model_list failed: {0}")] - ModelListParsing(#[source] serde_json::Error), -} diff --git a/litellm-rust/crates/config/src/lib.rs b/litellm-rust/crates/config/src/lib.rs deleted file mode 100644 index 655affbb0b7..00000000000 --- a/litellm-rust/crates/config/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod error; -#[cfg(feature = "python")] -mod python; - -pub use error::Error; -#[cfg(feature = "python")] -pub use python::load_model_list; diff --git a/litellm-rust/crates/config/src/python.rs b/litellm-rust/crates/config/src/python.rs deleted file mode 100644 index fdad5027baa..00000000000 --- a/litellm-rust/crates/config/src/python.rs +++ /dev/null @@ -1,76 +0,0 @@ -use std::path::Path; - -use litellm_core::router::Deployment; -use pyo3::prelude::*; - -use crate::Error; - -pub fn load_model_list(config_path: &Path) -> Result, Error> { - Python::attach(|python| { - let model_list = python - .import("litellm.proxy.read_model_list") - .and_then(|module| module.getattr("read_model_list")) - .and_then(|reader| reader.call1((config_path.to_string_lossy().as_ref(),))) - .map_err(|error| Error::PythonLoading(error.to_string()))?; - - let model_list_json = python - .import("json") - .and_then(|json| json.getattr("dumps")) - .and_then(|dumps| dumps.call1((model_list,))) - .and_then(|encoded| encoded.extract::()) - .map_err(|error| Error::Serialization(error.to_string()))?; - - parse_model_list(&model_list_json) - }) -} - -fn parse_model_list(model_list_json: &str) -> Result, Error> { - serde_json::from_str(model_list_json).map_err(Error::ModelListParsing) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parses_resolved_model_list() { - let deployments = parse_model_list( - r#"[ - { - "model_name": "realtime", - "litellm_params": { - "model": "openai/gpt-realtime", - "api_key": "resolved-secret", - "api_base": "https://api.example.test/v1" - } - }, - { - "model_name": "without-optional-values", - "litellm_params": {"model": "openai/gpt-4.1"} - } - ]"#, - ) - .expect("resolved model list should parse"); - - assert_eq!(deployments.len(), 2); - assert_eq!(deployments[0].model_name, "realtime"); - assert_eq!( - deployments[0].litellm_params.api_key.as_deref(), - Some("resolved-secret") - ); - assert_eq!( - deployments[0].litellm_params.api_base.as_deref(), - Some("https://api.example.test/v1") - ); - assert_eq!(deployments[1].litellm_params.api_key, None); - assert_eq!(deployments[1].litellm_params.api_base, None); - } - - #[test] - fn malformed_model_list_returns_parsing_error() { - let error = parse_model_list(r#"[{"model_name":"missing-params"}]"#) - .expect_err("missing litellm_params should fail"); - - assert!(matches!(error, Error::ModelListParsing(_))); - } -} diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 09c526f73cf..2b3b4c6a03f 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -28,8 +28,6 @@ subtle.workspace = true tokio = { workspace = true, features = ["sync"] } tokio-tungstenite.workspace = true thiserror.workspace = true -tracing.workspace = true -tracing-subscriber = { workspace = true, optional = true } sha2.workspace = true url.workspace = true veil.workspace = true @@ -50,8 +48,6 @@ bedrock-auth = [ "dep:aws-types", "dep:aws-smithy-runtime-api", ] -observability = ["dep:tracing-subscriber"] [dev-dependencies] rstest.workspace = true -tracing-subscriber.workspace = true diff --git a/litellm-rust/crates/core/src/audio_transcription/handler.rs b/litellm-rust/crates/core/src/audio_transcription/handler.rs index 9a96b9d1140..db665cf1faf 100644 --- a/litellm-rust/crates/core/src/audio_transcription/handler.rs +++ b/litellm-rust/crates/core/src/audio_transcription/handler.rs @@ -6,7 +6,6 @@ use crate::http_utils::{http_request, truncate_error_body}; use super::client::http_client; use super::types::ProviderAudioTranscriptionRequest; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub async fn execute_audio_transcription_provider_call( request: ProviderAudioTranscriptionRequest, ) -> Result { diff --git a/litellm-rust/crates/core/src/audio_transcription/mod.rs b/litellm-rust/crates/core/src/audio_transcription/mod.rs index 31b6de4b3e4..b71748082bf 100644 --- a/litellm-rust/crates/core/src/audio_transcription/mod.rs +++ b/litellm-rust/crates/core/src/audio_transcription/mod.rs @@ -11,7 +11,6 @@ pub use handler::execute_audio_transcription_provider_call; pub use prepare::prepare_audio_transcription_provider_call; pub use types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest}; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result { execute_audio_transcription_provider_call(prepare_audio_transcription_provider_call(request)?) .await diff --git a/litellm-rust/crates/core/src/audio_transcription/prepare.rs b/litellm-rust/crates/core/src/audio_transcription/prepare.rs index bbef97341a9..9f45b8804be 100644 --- a/litellm-rust/crates/core/src/audio_transcription/prepare.rs +++ b/litellm-rust/crates/core/src/audio_transcription/prepare.rs @@ -2,12 +2,11 @@ use crate::error::Error; use crate::http_utils::{has_header, string_headers}; #[cfg(feature = "bedrock-auth")] use crate::providers::bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG; -use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; +use crate::providers::custom_llm_provider::{CustomLlmProvider, get_custom_llm_provider}; use super::transformation::{AudioTranscriptionAuth, AudioTranscriptionProviderConfig}; use super::types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest}; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn provider_config(provider: &str) -> Option<&'static dyn AudioTranscriptionProviderConfig> { #[cfg(feature = "bedrock-auth")] if provider == "bedrock" { @@ -17,7 +16,6 @@ fn provider_config(provider: &str) -> Option<&'static dyn AudioTranscriptionProv None } -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub fn prepare_audio_transcription_provider_call( request: AudioTranscriptionRequest<'_>, ) -> Result { diff --git a/litellm-rust/crates/core/src/audio_transcription/transformation.rs b/litellm-rust/crates/core/src/audio_transcription/transformation.rs index aa9846427dc..16a28fbcac0 100644 --- a/litellm-rust/crates/core/src/audio_transcription/transformation.rs +++ b/litellm-rust/crates/core/src/audio_transcription/transformation.rs @@ -15,7 +15,6 @@ pub enum AudioTranscriptionAuth { pub trait AudioTranscriptionProviderConfig: Sync { fn supported_transcription_params(&self) -> &'static [&'static str]; - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn map_transcription_params(&self, params: &Map) -> Map { params .iter() diff --git a/litellm-rust/crates/core/src/auth/vertex.rs b/litellm-rust/crates/core/src/auth/vertex.rs index 00a0a7ea7ee..e13b57abdfe 100644 --- a/litellm-rust/crates/core/src/auth/vertex.rs +++ b/litellm-rust/crates/core/src/auth/vertex.rs @@ -106,7 +106,6 @@ impl VertexAuth { } } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(crate) async fn validate_environment( &self, headers: Vec<(String, String)>, diff --git a/litellm-rust/crates/core/src/chat_completions/common_utils.rs b/litellm-rust/crates/core/src/chat_completions/common_utils.rs index 69e5f175ad5..ca51471eb7c 100644 --- a/litellm-rust/crates/core/src/chat_completions/common_utils.rs +++ b/litellm-rust/crates/core/src/chat_completions/common_utils.rs @@ -7,7 +7,6 @@ use super::transformation::ChatCompletionsProviderConfig; const HEADER_CONTEXT: &str = "chat completions"; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(super) fn chat_completions_provider_config( provider: &str, ) -> Option<&'static dyn ChatCompletionsProviderConfig> { diff --git a/litellm-rust/crates/core/src/chat_completions/handler.rs b/litellm-rust/crates/core/src/chat_completions/handler.rs index 96d001e2892..799dd3ceeb0 100644 --- a/litellm-rust/crates/core/src/chat_completions/handler.rs +++ b/litellm-rust/crates/core/src/chat_completions/handler.rs @@ -11,7 +11,6 @@ use super::types::{ ResolvedChatCompletionsRequest, }; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(super) async fn execute_chat_completions_provider_call( request: ResolvedChatCompletionsRequest<'_>, ) -> Result { diff --git a/litellm-rust/crates/core/src/chat_completions/mod.rs b/litellm-rust/crates/core/src/chat_completions/mod.rs index 32dea17d202..8117663a23b 100644 --- a/litellm-rust/crates/core/src/chat_completions/mod.rs +++ b/litellm-rust/crates/core/src/chat_completions/mod.rs @@ -22,7 +22,6 @@ use handler::execute_chat_completions_provider_call; use prepare::{parse_messages, resolve_provider_config, resolve_request}; use types::{ChatCompletionsRequest, ChatCompletionsResponse}; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub async fn chat_completions( request: ChatCompletionsRequest<'_>, ) -> Result { diff --git a/litellm-rust/crates/core/src/chat_completions/prepare.rs b/litellm-rust/crates/core/src/chat_completions/prepare.rs index 3be2ba21de4..e965a0fe5d7 100644 --- a/litellm-rust/crates/core/src/chat_completions/prepare.rs +++ b/litellm-rust/crates/core/src/chat_completions/prepare.rs @@ -2,7 +2,7 @@ use serde_json::Value; use crate::error::Error; use crate::http_utils::has_header; -use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; +use crate::providers::custom_llm_provider::{CustomLlmProvider, get_custom_llm_provider}; use super::common_utils::{chat_completions_provider_config, string_headers}; use super::transformation::{ChatCompletionsAuth, ChatCompletionsProviderConfig}; @@ -62,7 +62,6 @@ pub(super) fn resolve_request( }) } -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn validate_environment( request: &ResolvedChatCompletionsRequest<'_>, model: &str, diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/core/src/constants.rs index 1babb0078b8..4ff4333c4ac 100644 --- a/litellm-rust/crates/core/src/constants.rs +++ b/litellm-rust/crates/core/src/constants.rs @@ -42,8 +42,6 @@ pub const CHAT_COMPLETION_OBJECT: &str = "chat.completion"; pub const EMPTY_TEXT_PLACEHOLDER: &str = "[System: Empty message content sanitised to satisfy protocol]"; -pub const FUNCTION_TRACE_TARGET: &str = "litellm::function_trace"; - pub(crate) const MEDIA_CONNECT_TIMEOUT_SECS: u64 = 10; pub(crate) const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024; diff --git a/litellm-rust/crates/core/src/http_utils.rs b/litellm-rust/crates/core/src/http_utils.rs index 9299bb77ac8..7b9d6f6a648 100644 --- a/litellm-rust/crates/core/src/http_utils.rs +++ b/litellm-rust/crates/core/src/http_utils.rs @@ -38,7 +38,6 @@ pub(crate) fn with_headers( }) } -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub async fn http_request( request: reqwest::RequestBuilder, ) -> Result { diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index 0b3573deab2..f1d888ec12c 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -8,14 +8,9 @@ pub mod error; pub mod http_utils; mod media; pub mod messages; -#[cfg(any(feature = "observability", test))] -pub mod observability; pub mod ocr; pub mod providers; -pub mod realtime; pub mod responses; -pub mod router; -pub mod routing_utils; mod url_utils; pub use auth::AuthError; diff --git a/litellm-rust/crates/core/src/messages/common_utils.rs b/litellm-rust/crates/core/src/messages/common_utils.rs index 8f0f6652fa4..8dfdb2e361a 100644 --- a/litellm-rust/crates/core/src/messages/common_utils.rs +++ b/litellm-rust/crates/core/src/messages/common_utils.rs @@ -10,7 +10,6 @@ pub(super) use crate::http_utils::{has_bearer_auth, has_header, truncate_error_b const HEADER_CONTEXT: &str = "messages"; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(super) fn messages_provider_config( provider: &str, ) -> Option<&'static dyn AnthropicMessagesProviderConfig> { diff --git a/litellm-rust/crates/core/src/messages/handler.rs b/litellm-rust/crates/core/src/messages/handler.rs index 61ff81bcdc8..4d2627c6383 100644 --- a/litellm-rust/crates/core/src/messages/handler.rs +++ b/litellm-rust/crates/core/src/messages/handler.rs @@ -7,7 +7,6 @@ use super::common_utils::truncate_error_body; use super::prepare::prepare_provider_request; use super::types::{AnthropicMessagesResponse, MessagesRequest}; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(super) async fn execute_messages_provider_call( request: MessagesRequest<'_>, ) -> Result { diff --git a/litellm-rust/crates/core/src/messages/mod.rs b/litellm-rust/crates/core/src/messages/mod.rs index cfa8bda1104..0083272bcb4 100644 --- a/litellm-rust/crates/core/src/messages/mod.rs +++ b/litellm-rust/crates/core/src/messages/mod.rs @@ -18,7 +18,6 @@ pub mod types; use handler::{execute_messages_provider_call, execute_messages_provider_stream}; use types::{AnthropicMessagesResponse, MessagesRequest}; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub async fn messages(request: MessagesRequest<'_>) -> Result { execute_messages_provider_call(request).await } diff --git a/litellm-rust/crates/core/src/messages/prepare.rs b/litellm-rust/crates/core/src/messages/prepare.rs index ec83d03f535..3c2f6b9d372 100644 --- a/litellm-rust/crates/core/src/messages/prepare.rs +++ b/litellm-rust/crates/core/src/messages/prepare.rs @@ -1,5 +1,5 @@ use crate::error::Error; -use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; +use crate::providers::custom_llm_provider::{CustomLlmProvider, get_custom_llm_provider}; use super::common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers}; use super::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; @@ -56,7 +56,6 @@ pub(super) fn prepare_provider_request( }) } -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn validate_environment( config: &dyn AnthropicMessagesProviderConfig, extra_headers: Option>, diff --git a/litellm-rust/crates/core/src/messages/transformation.rs b/litellm-rust/crates/core/src/messages/transformation.rs index a5904c085a0..673a5728aca 100644 --- a/litellm-rust/crates/core/src/messages/transformation.rs +++ b/litellm-rust/crates/core/src/messages/transformation.rs @@ -45,7 +45,6 @@ pub trait AnthropicMessagesProviderConfig: Sync { ] } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_request( &self, request: AnthropicMessagesRequest, @@ -53,7 +52,6 @@ pub trait AnthropicMessagesProviderConfig: Sync { Ok(request) } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_response( &self, _model: &str, diff --git a/litellm-rust/crates/core/src/observability/function_trace.rs b/litellm-rust/crates/core/src/observability/function_trace.rs deleted file mode 100644 index 2031e35901c..00000000000 --- a/litellm-rust/crates/core/src/observability/function_trace.rs +++ /dev/null @@ -1,215 +0,0 @@ -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; - -use serde::Serialize; -use tracing::span::{Attributes, Id}; -use tracing::{Dispatch, Subscriber}; -use tracing_subscriber::layer::Context; -use tracing_subscriber::prelude::*; -use tracing_subscriber::registry::LookupSpan; -use tracing_subscriber::{Layer, Registry}; - -use super::function_trace_filter; - -#[derive(Clone, Debug, PartialEq, Serialize)] -pub struct FunctionTraceEvent { - pub id: usize, - pub parent_id: Option, - pub function: &'static str, - pub module_path: Option<&'static str>, - pub file: Option<&'static str>, - pub line: Option, -} - -#[derive(Clone, Default)] -pub struct FunctionTrace { - events: Arc>>, - span_events: Arc>>, -} - -impl FunctionTrace { - pub fn dispatcher(&self) -> Dispatch { - Dispatch::new( - Registry::default().with( - FunctionTraceLayer { - trace: self.clone(), - } - .with_filter(function_trace_filter()), - ), - ) - } - - pub fn events(&self) -> Vec { - self.events - .lock() - .unwrap_or_else(|error| error.into_inner()) - .clone() - } -} - -struct FunctionTraceLayer { - trace: FunctionTrace, -} - -impl Layer for FunctionTraceLayer -where - S: Subscriber + for<'lookup> LookupSpan<'lookup>, -{ - fn on_new_span(&self, attributes: &Attributes<'_>, id: &Id, context: Context<'_, S>) { - let parent_id = context.span(id).and_then(|span| { - let span_events = self - .trace - .span_events - .lock() - .unwrap_or_else(|error| error.into_inner()); - span.scope() - .skip(1) - .find_map(|ancestor| span_events.get(&ancestor.id()).copied()) - }); - let mut events = self - .trace - .events - .lock() - .unwrap_or_else(|error| error.into_inner()); - let event_id = events.len(); - events.push(FunctionTraceEvent { - id: event_id, - parent_id, - function: attributes.metadata().name(), - module_path: attributes.metadata().module_path(), - file: attributes.metadata().file(), - line: attributes.metadata().line(), - }); - self.trace - .span_events - .lock() - .unwrap_or_else(|error| error.into_inner()) - .insert(id.clone(), event_id); - } -} - -#[cfg(test)] -mod tests { - use crate::constants::FUNCTION_TRACE_TARGET; - - use super::*; - - fn event( - id: usize, - parent_id: Option, - function: &'static str, - ) -> (usize, Option, &'static str) { - (id, parent_id, function) - } - - fn structural_events( - events: &[FunctionTraceEvent], - ) -> Vec<(usize, Option, &'static str)> { - events - .iter() - .map(|event| (event.id, event.parent_id, event.function)) - .collect() - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - async fn outer() { - tokio::task::yield_now().await; - inner().await; - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - async fn inner() { - tokio::task::yield_now().await; - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - async fn concurrent_parent() { - tokio::join!(inner(), inner()); - } - - #[tokio::test] - async fn concurrent_futures_keep_separate_traces_across_yields() { - use tracing::instrument::WithSubscriber; - - let first = FunctionTrace::default(); - let second = FunctionTrace::default(); - let outside = FunctionTrace::default(); - - async { - tokio::join!( - outer().with_subscriber(first.dispatcher()), - inner().with_subscriber(second.dispatcher()), - ); - inner().await; - } - .with_subscriber(outside.dispatcher()) - .await; - - assert_eq!( - structural_events(&first.events()), - vec![event(0, None, "outer"), event(1, Some(0), "inner")], - ); - assert_eq!( - structural_events(&second.events()), - vec![event(0, None, "inner")], - ); - assert_eq!( - structural_events(&outside.events()), - vec![event(0, None, "inner")], - ); - } - - #[tokio::test] - async fn concurrent_siblings_keep_the_same_parent() { - use tracing::instrument::WithSubscriber; - - let trace = FunctionTrace::default(); - concurrent_parent() - .with_subscriber(trace.dispatcher()) - .await; - - assert_eq!( - structural_events(&trace.events()), - vec![ - event(0, None, "concurrent_parent"), - event(1, Some(0), "inner"), - event(2, Some(0), "inner"), - ] - ); - } - - #[test] - fn records_matching_spans_in_creation_order() { - let trace = FunctionTrace::default(); - let dispatch = trace.dispatcher(); - - tracing::dispatcher::with_default(&dispatch, || { - let _ignored = tracing::trace_span!(target: "other", "ignored"); - let _first = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name"); - let _wrong_level = tracing::debug_span!(target: FUNCTION_TRACE_TARGET, "wrong_level"); - let _second = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "same_name"); - }); - - assert_eq!( - structural_events(&trace.events()), - vec![event(0, None, "same_name"), event(1, None, "same_name")] - ); - } - - #[test] - fn records_matching_span_nesting_depth() { - let trace = FunctionTrace::default(); - let dispatch = trace.dispatcher(); - - tracing::dispatcher::with_default(&dispatch, || { - let outer = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "outer"); - let _outer_guard = outer.enter(); - let _inner = tracing::trace_span!(target: FUNCTION_TRACE_TARGET, "inner"); - }); - - assert_eq!( - structural_events(&trace.events()), - vec![event(0, None, "outer"), event(1, Some(0), "inner")] - ); - } -} diff --git a/litellm-rust/crates/core/src/observability/mod.rs b/litellm-rust/crates/core/src/observability/mod.rs deleted file mode 100644 index 3f9da8e2bb4..00000000000 --- a/litellm-rust/crates/core/src/observability/mod.rs +++ /dev/null @@ -1,59 +0,0 @@ -use tracing::span::Id; -use tracing::{Level, Metadata, Subscriber}; -use tracing_subscriber::filter::{FilterFn, LevelFilter, filter_fn}; -use tracing_subscriber::layer::Context; -use tracing_subscriber::registry::LookupSpan; - -use crate::constants::FUNCTION_TRACE_TARGET; - -pub mod function_trace; - -pub use function_trace::{FunctionTrace, FunctionTraceEvent}; - -pub fn function_trace_filter() -> FilterFn) -> bool> { - filter_fn(|metadata| { - metadata.is_span() - && metadata.target() == FUNCTION_TRACE_TARGET - && *metadata.level() == Level::TRACE - }) - .with_max_level_hint(LevelFilter::TRACE) -} - -pub fn span_depth(context: &Context<'_, S>, id: &Id) -> usize -where - S: Subscriber + for<'lookup> LookupSpan<'lookup>, -{ - context - .span(id) - .map(|span| span.scope().skip(1).count()) - .unwrap_or_default() -} - -#[cfg(test)] -mod tests { - use tracing::instrument::WithSubscriber; - - use super::*; - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - async fn instrumented_with_literal_target() {} - - #[tokio::test] - async fn literal_instrument_target_matches_filter_constant() { - assert_eq!(FUNCTION_TRACE_TARGET, "litellm::function_trace"); - - let trace = FunctionTrace::default(); - instrumented_with_literal_target() - .with_subscriber(trace.dispatcher()) - .await; - - let events = trace.events(); - assert_eq!(events.len(), 1); - assert_eq!(events[0].id, 0); - assert_eq!(events[0].parent_id, None); - assert_eq!(events[0].function, "instrumented_with_literal_target"); - assert_eq!(events[0].module_path, Some(module_path!())); - assert_eq!(events[0].file, Some(file!())); - assert!(events[0].line.is_some()); - } -} diff --git a/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/mod.rs b/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/mod.rs index e90c27ba59d..22034ddc3f6 100644 --- a/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/mod.rs +++ b/litellm-rust/crates/core/src/ocr/adapters/azure/document_intelligence/mod.rs @@ -75,7 +75,6 @@ impl OcrAdapter for AzureDocumentIntelligenceAdapter { } } -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn map_ocr_params( request: &LiteLLMOcrRequest, ) -> Result { diff --git a/litellm-rust/crates/core/src/ocr/client.rs b/litellm-rust/crates/core/src/ocr/client.rs index 394ca778d2f..041c5c6fd21 100644 --- a/litellm-rust/crates/core/src/ocr/client.rs +++ b/litellm-rust/crates/core/src/ocr/client.rs @@ -36,12 +36,6 @@ impl OcrClient { shared_client() } - #[tracing::instrument( - name = "ocr", - target = "litellm::function_trace", - level = "trace", - skip_all - )] pub async fn perform(&self, request: LiteLLMOcrRequest) -> Result { use super::{ NativeOutcome, OcrAdmission, OcrCall, OcrCallStep, OcrHookHost, OcrHost, diff --git a/litellm-rust/crates/core/src/ocr/codecs/deepseek/transformation.rs b/litellm-rust/crates/core/src/ocr/codecs/deepseek/transformation.rs index 7e8ce63b379..999ac6cf032 100644 --- a/litellm-rust/crates/core/src/ocr/codecs/deepseek/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/codecs/deepseek/transformation.rs @@ -5,7 +5,6 @@ use super::types::*; use crate::ocr::error::{OcrRequestError, OcrResponseError}; use crate::ocr::types::{LiteLLMOcrResponse, OcrDocument}; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(crate) fn transform_ocr_request( provider_model: &str, document: OcrDocument, diff --git a/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/transformation.rs b/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/transformation.rs index f76a7c2b232..018d7eb9c65 100644 --- a/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/codecs/document_intelligence/transformation.rs @@ -7,7 +7,6 @@ use crate::ocr::document::InlineDocument; use crate::ocr::error::{OcrRequestError, OcrResponseError}; use crate::ocr::types::{LiteLLMOcrResponse, OcrDocument}; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(crate) fn transform_ocr_request( document: OcrDocument, ) -> Result { diff --git a/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs b/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs index e60f1f5d3d6..e8073905548 100644 --- a/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/codecs/mistral/transformation.rs @@ -2,7 +2,6 @@ use super::{MistralOcrParams, MistralOcrRequest, MistralOcrResponse}; use crate::ocr::error::{OcrRequestError, OcrResponseError}; use crate::ocr::types::{LiteLLMOcrResponse, OcrDocument}; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(crate) fn transform_ocr_request( model: &str, document: OcrDocument, diff --git a/litellm-rust/crates/core/src/ocr/codecs/reducto/transformation.rs b/litellm-rust/crates/core/src/ocr/codecs/reducto/transformation.rs index 7073643f6b6..f4c8338c134 100644 --- a/litellm-rust/crates/core/src/ocr/codecs/reducto/transformation.rs +++ b/litellm-rust/crates/core/src/ocr/codecs/reducto/transformation.rs @@ -6,12 +6,6 @@ use super::types::*; use crate::ocr::error::{OcrRequestError, OcrResponseError}; use crate::ocr::types::{LiteLLMOcrResponse, OcrDocument}; -#[tracing::instrument( - name = "transform_ocr_request", - target = "litellm::function_trace", - level = "trace", - skip_all -)] pub(crate) fn transform_v3_ocr_request( _model: &str, document: OcrDocument, @@ -23,12 +17,6 @@ pub(crate) fn transform_v3_ocr_request( }) } -#[tracing::instrument( - name = "transform_ocr_request", - target = "litellm::function_trace", - level = "trace", - skip_all -)] pub(crate) fn transform_legacy_ocr_request( _model: &str, document: OcrDocument, diff --git a/litellm-rust/crates/core/src/ocr/hooks.rs b/litellm-rust/crates/core/src/ocr/hooks.rs index 3e7507e9ed5..4d4a387e553 100644 --- a/litellm-rust/crates/core/src/ocr/hooks.rs +++ b/litellm-rust/crates/core/src/ocr/hooks.rs @@ -125,12 +125,6 @@ impl CallLifecycleHooks( &'a self, context: &'a CallLifecycleContext, @@ -140,12 +134,6 @@ impl CallLifecycleHooks( &'a self, context: &'a CallLifecycleContext, diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/core/src/ocr/prepare.rs index 9934a1d9a14..241cb09529a 100644 --- a/litellm-rust/crates/core/src/ocr/prepare.rs +++ b/litellm-rust/crates/core/src/ocr/prepare.rs @@ -14,7 +14,6 @@ pub(crate) struct ParsedProviderParams { pub extra_params: Map, } -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(crate) fn _prepare_ocr_request( request: &LiteLLMOcrRequest, ) -> Result, OcrRequestError> { diff --git a/litellm-rust/crates/core/src/ocr/registry.rs b/litellm-rust/crates/core/src/ocr/registry.rs index ed7d4fd5cf2..54f57cc3ca2 100644 --- a/litellm-rust/crates/core/src/ocr/registry.rs +++ b/litellm-rust/crates/core/src/ocr/registry.rs @@ -1,6 +1,6 @@ use super::adapters::OcrAdapter; use crate::Error; -use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; +use crate::providers::custom_llm_provider::{CustomLlmProvider, get_custom_llm_provider}; macro_rules! define_adapter_types { ($( $variant:ident, $adapter:ty, $instance:expr, $provider:ident; )+) => { diff --git a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs index a7d5a8ad0cf..56175322442 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs @@ -117,7 +117,6 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { }) } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn supported_openai_params(&self) -> &'static [(&'static str, &'static str)] { SUPPORTED_PARAMS } @@ -138,7 +137,6 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { }) } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_request( &self, model: &str, @@ -150,7 +148,6 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { }) } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_response( &self, _model: &str, diff --git a/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs b/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs index 3ed00b7cc5f..fe96155bbbb 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs @@ -42,7 +42,6 @@ pub fn complete_anthropic_url( } impl AnthropicMessagesProviderConfig for AnthropicMessagesConfig { - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn complete_url( &self, api_base: Option<&str>, diff --git a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs index 585b34f393f..8edda1901e5 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs @@ -132,7 +132,6 @@ fn fold_system_role_messages(request: AnthropicMessagesRequest) -> AnthropicMess } impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn complete_url( &self, api_base: Option<&str>, diff --git a/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs b/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs index 9bf1f73a74d..bb4f6afe5f9 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs @@ -46,12 +46,10 @@ fn optional_string<'a>(params: &'a Map, key: &str) -> Option<&'a } impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn supported_transcription_params(&self) -> &'static [&'static str] { SUPPORTED_PARAMS } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_transcription_request( &self, _model: &str, @@ -85,7 +83,6 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { }) } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_transcription_response( &self, _model: &str, diff --git a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs index 7be3d108d44..02e6797b338 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs @@ -163,7 +163,6 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { &[("Content-Type", "application/json")] } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn supported_openai_params(&self) -> &'static [(&'static str, &'static str)] { SUPPORTED_PARAMS } diff --git a/litellm-rust/crates/core/src/routing_utils/provider.rs b/litellm-rust/crates/core/src/providers/custom_llm_provider.rs similarity index 100% rename from litellm-rust/crates/core/src/routing_utils/provider.rs rename to litellm-rust/crates/core/src/providers/custom_llm_provider.rs diff --git a/litellm-rust/crates/core/src/providers/mod.rs b/litellm-rust/crates/core/src/providers/mod.rs index 1aeb75063d6..6b1881d56ca 100644 --- a/litellm-rust/crates/core/src/providers/mod.rs +++ b/litellm-rust/crates/core/src/providers/mod.rs @@ -2,4 +2,5 @@ pub mod anthropic; pub mod azure_ai; #[cfg(feature = "bedrock-auth")] pub mod bedrock; +pub mod custom_llm_provider; pub mod openai; diff --git a/litellm-rust/crates/core/src/providers/openai/mod.rs b/litellm-rust/crates/core/src/providers/openai/mod.rs index 62fcc50f2ac..b396b037bc5 100644 --- a/litellm-rust/crates/core/src/providers/openai/mod.rs +++ b/litellm-rust/crates/core/src/providers/openai/mod.rs @@ -1,2 +1 @@ -pub mod realtime; pub mod responses; diff --git a/litellm-rust/crates/core/src/providers/openai/realtime/mod.rs b/litellm-rust/crates/core/src/providers/openai/realtime/mod.rs deleted file mode 100644 index f239b6921fa..00000000000 --- a/litellm-rust/crates/core/src/providers/openai/realtime/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod transformation; diff --git a/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs b/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs deleted file mode 100644 index f1985f81b7d..00000000000 --- a/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs +++ /dev/null @@ -1,189 +0,0 @@ -use crate::Error; -use crate::realtime::transformation::RealtimeProviderConfig; -use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult}; - -/// Default OpenAI API base, used when the caller does not override `api_base`. -pub const OPENAI_REALTIME_DEFAULT_API_BASE: &str = "https://api.openai.com"; - -/// Path appended to the resolved host base to reach the realtime endpoint. -pub const OPENAI_REALTIME_PATH: &str = "/v1/realtime"; - -/// Percent-encode a query value, escaping any char outside the RFC 3986 -/// unreserved set (`A-Za-z0-9-._~`). Keeps us dependency-free; common realtime -/// model slugs have no special chars, but this stays correct for the rest. -fn percent_encode(value: &str) -> String { - let mut encoded = String::with_capacity(value.len()); - for byte in value.bytes() { - let unreserved = byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~'); - if unreserved { - encoded.push(byte as char); - } else { - encoded.push('%'); - encoded.push_str(&format!("{byte:02X}")); - } - } - encoded -} - -/// Build the realtime WebSocket URL, porting Python's `OpenAIRealtime._construct_url`. -/// -/// Blank/whitespace `api_base` is treated as absent (guard at resolution time), -/// falling back to the default. The scheme is swapped to its WebSocket -/// equivalent (`https://`→`wss://`, `http://`→`ws://`); bases already using -/// `ws`/`wss` are left untouched. A bare host or unrecognized scheme defaults to -/// secure `wss://` so we never hand a scheme-less URL to the connector (this is -/// a deliberate hardening over Python's `_construct_url`, which would emit a -/// scheme-less URL here). A trailing `/` is trimmed before the path and -/// `?model=` are appended. -pub fn complete_url(api_base: Option<&str>, model: &str) -> String { - let base = api_base - .map(str::trim) - .filter(|base| !base.is_empty()) - .unwrap_or(OPENAI_REALTIME_DEFAULT_API_BASE); - - let base = if let Some(rest) = base.strip_prefix("https://") { - format!("wss://{rest}") - } else if let Some(rest) = base.strip_prefix("http://") { - format!("ws://{rest}") - } else if base.starts_with("wss://") || base.starts_with("ws://") { - base.to_string() - } else { - format!("wss://{base}") - }; - - let base = base.trim_end_matches('/'); - - format!( - "{base}{OPENAI_REALTIME_PATH}?model={}", - percent_encode(model) - ) -} - -pub struct OpenAiRealtimeConfig; - -pub const OPENAI_REALTIME_CONFIG: OpenAiRealtimeConfig = OpenAiRealtimeConfig; - -impl RealtimeProviderConfig for OpenAiRealtimeConfig { - fn complete_url(&self, api_base: Option<&str>, model: &str) -> String { - complete_url(api_base, model) - } - - fn transform_realtime_request( - &self, - event: &RealtimeEvent, - _model: &str, - ) -> Result { - Ok(RealtimeTransformResult::passthrough(event.clone())) - } - - fn transform_realtime_response( - &self, - event: &RealtimeEvent, - _model: &str, - ) -> Result { - Ok(RealtimeTransformResult::passthrough(event.clone())) - } -} - -pub fn transform_realtime_request( - event: &RealtimeEvent, - model: &str, -) -> Result { - OPENAI_REALTIME_CONFIG.transform_realtime_request(event, model) -} - -pub fn transform_realtime_response( - event: &RealtimeEvent, - model: &str, -) -> Result { - OPENAI_REALTIME_CONFIG.transform_realtime_response(event, model) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn complete_url_defaults_to_openai_wss() { - assert_eq!( - complete_url(None, "gpt-4o-realtime-preview"), - "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview" - ); - } - - #[test] - fn complete_url_blank_base_uses_default() { - assert_eq!( - complete_url(Some(" "), "gpt-4o-realtime-preview"), - "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview" - ); - } - - #[test] - fn complete_url_swaps_http_to_ws() { - assert_eq!( - complete_url(Some("http://localhost:8080"), "gpt-4o-realtime-preview"), - "ws://localhost:8080/v1/realtime?model=gpt-4o-realtime-preview" - ); - } - - #[test] - fn complete_url_dedupes_trailing_slash() { - assert_eq!( - complete_url(Some("https://api.openai.com/"), "gpt-4o-realtime-preview"), - "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview" - ); - } - - #[test] - fn complete_url_custom_base() { - assert_eq!( - complete_url(Some("https://oai.azure.example"), "gpt-4o-realtime-preview"), - "wss://oai.azure.example/v1/realtime?model=gpt-4o-realtime-preview" - ); - } - - #[test] - fn complete_url_preserves_existing_wss_scheme() { - assert_eq!( - complete_url(Some("wss://api.openai.com"), "gpt-realtime"), - "wss://api.openai.com/v1/realtime?model=gpt-realtime" - ); - } - - #[test] - fn complete_url_bare_host_defaults_to_wss() { - assert_eq!( - complete_url(Some("api.openai.com"), "gpt-realtime"), - "wss://api.openai.com/v1/realtime?model=gpt-realtime" - ); - } - - #[test] - fn complete_url_percent_encodes_model_space() { - assert_eq!( - complete_url(None, "gpt 4o"), - "wss://api.openai.com/v1/realtime?model=gpt%204o" - ); - } - - #[test] - fn transform_realtime_request_passthrough_preserves_event() { - let event: RealtimeEvent = - serde_json::from_str(r#"{"type":"session.update","session":{"voice":"alloy"}}"#) - .expect("valid event"); - let result = - transform_realtime_request(&event, "gpt-realtime").expect("passthrough is infallible"); - assert_eq!(result.events, vec![event]); - } - - #[test] - fn transform_realtime_response_passthrough_preserves_event() { - let event: RealtimeEvent = - serde_json::from_str(r#"{"type":"response.output_audio.delta","delta":"abc=="}"#) - .expect("valid event"); - let result = - transform_realtime_response(&event, "gpt-realtime").expect("passthrough is infallible"); - assert_eq!(result.events, vec![event]); - } -} diff --git a/litellm-rust/crates/core/src/realtime/mod.rs b/litellm-rust/crates/core/src/realtime/mod.rs deleted file mode 100644 index ec2fbb969a6..00000000000 --- a/litellm-rust/crates/core/src/realtime/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod transformation; -pub mod types; diff --git a/litellm-rust/crates/core/src/realtime/transformation.rs b/litellm-rust/crates/core/src/realtime/transformation.rs deleted file mode 100644 index b08084514ef..00000000000 --- a/litellm-rust/crates/core/src/realtime/transformation.rs +++ /dev/null @@ -1,22 +0,0 @@ -use crate::Error; -use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult}; - -pub trait RealtimeProviderConfig { - /// Build the upstream WebSocket URL (e.g. `wss://api.openai.com/v1/realtime?model=…`). - /// Pure string construction only — no network, no env. - fn complete_url(&self, api_base: Option<&str>, model: &str) -> String; - - /// Transform a client → backend event before it is forwarded upstream. - fn transform_realtime_request( - &self, - event: &RealtimeEvent, - model: &str, - ) -> Result; - - /// Transform a backend → client event before it is forwarded downstream. - fn transform_realtime_response( - &self, - event: &RealtimeEvent, - model: &str, - ) -> Result; -} diff --git a/litellm-rust/crates/core/src/realtime/types.rs b/litellm-rust/crates/core/src/realtime/types.rs deleted file mode 100644 index 3b59224b6e9..00000000000 --- a/litellm-rust/crates/core/src/realtime/types.rs +++ /dev/null @@ -1,60 +0,0 @@ -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; - -/// A single realtime event exchanged over the WebSocket. -/// -/// The `type` discriminator is a typed field; the remaining fields are -/// preserved losslessly in `data` so a transform can pass an event through, or -/// inspect/modify specific fields, without enumerating every event variant. -/// Wire (de)serialization happens at the host edge — `core`/`providers` operate -/// only on this typed form. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct RealtimeEvent { - #[serde(rename = "type")] - pub event_type: String, - #[serde(flatten)] - pub data: Map, -} - -/// One or more typed events produced by a realtime transform. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct RealtimeTransformResult { - pub events: Vec, -} - -impl RealtimeTransformResult { - /// Forward a single event unchanged (the OpenAI baseline). - pub fn passthrough(event: RealtimeEvent) -> Self { - Self { - events: vec![event], - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn event(raw: &str) -> RealtimeEvent { - serde_json::from_str(raw).expect("valid event json") - } - - #[test] - fn realtime_event_round_trips_type_and_extra_fields() { - let raw = r#"{"type":"response.output_text.delta","delta":"hi","response_id":"r1"}"#; - let parsed = event(raw); - assert_eq!(parsed.event_type, "response.output_text.delta"); - assert_eq!(parsed.data.get("delta"), Some(&Value::String("hi".into()))); - // Re-serializing yields a semantically-equal event (key order may differ). - let reparsed: RealtimeEvent = - serde_json::from_str(&serde_json::to_string(&parsed).unwrap()).unwrap(); - assert_eq!(parsed, reparsed); - } - - #[test] - fn passthrough_produces_single_element_vec() { - let parsed = event(r#"{"type":"session.update"}"#); - let result = RealtimeTransformResult::passthrough(parsed.clone()); - assert_eq!(result.events, vec![parsed]); - } -} diff --git a/litellm-rust/crates/core/src/router/deployment.rs b/litellm-rust/crates/core/src/router/deployment.rs deleted file mode 100644 index 1ee88e682a3..00000000000 --- a/litellm-rust/crates/core/src/router/deployment.rs +++ /dev/null @@ -1,44 +0,0 @@ -//! `model_list` data types, mirroring Python's deployment dict. Deserialize-ready -//! so a deployment can be loaded straight from the proxy config's `model_list`. - -use serde::Deserialize; - -/// Per-deployment call parameters, mirroring Python's `litellm_params`. -#[derive(Clone, Debug, Deserialize)] -pub struct LiteLLMParams { - /// Provider model, e.g. `gpt-realtime` or `openai/gpt-realtime`. - pub model: String, - #[serde(default)] - pub api_key: Option, - #[serde(default)] - pub api_base: Option, -} - -/// One entry of the `model_list`, mirroring Python's deployment dict. -#[derive(Clone, Debug, Deserialize)] -pub struct Deployment { - /// Public alias clients request, e.g. `gpt-realtime`. - pub model_name: String, - pub litellm_params: LiteLLMParams, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn deserializes_from_model_list_entry() { - let entry = r#"{ - "model_name": "gpt-realtime", - "litellm_params": {"model": "openai/gpt-realtime", "api_base": "https://x"} - }"#; - let deployment: Deployment = serde_json::from_str(entry).expect("valid entry"); - assert_eq!(deployment.model_name, "gpt-realtime"); - assert_eq!(deployment.litellm_params.model, "openai/gpt-realtime"); - assert_eq!(deployment.litellm_params.api_key, None); - assert_eq!( - deployment.litellm_params.api_base.as_deref(), - Some("https://x") - ); - } -} diff --git a/litellm-rust/crates/core/src/router/mod.rs b/litellm-rust/crates/core/src/router/mod.rs deleted file mode 100644 index 96bc91bc6b5..00000000000 --- a/litellm-rust/crates/core/src/router/mod.rs +++ /dev/null @@ -1,93 +0,0 @@ -//! Minimal Rust port of LiteLLM's `router.py` deployment selection. -//! -//! A [`Router`] is built from a `model_list` of [`Deployment`]s -//! (`{ model_name, litellm_params: { model, api_key, api_base } }`) and selects -//! one per request via a [`RoutingStrategy`]. For now the only strategy is -//! `simple-shuffle` — a uniform random pick within a `model_name` group. -//! -//! This stays pure (no I/O): it only *chooses* a deployment. The host (the -//! gateway) takes the chosen deployment and performs the actual provider call. -//! -//! - [`deployment`] — the `model_list` data types. -//! - [`strategy`] — how a deployment is chosen. - -mod deployment; -mod strategy; - -pub use deployment::{Deployment, LiteLLMParams}; -pub use strategy::RoutingStrategy; - -/// Load-balancing router over a `model_list`. -#[derive(Clone, Debug, Default)] -pub struct Router { - model_list: Vec, - routing_strategy: RoutingStrategy, -} - -impl Router { - /// Build a router from a `model_list` using the default `simple-shuffle` strategy. - pub fn new(model_list: Vec) -> Self { - Self { - model_list, - routing_strategy: RoutingStrategy::SimpleShuffle, - } - } - - /// All deployments in the `model_list`. Read-only; used by the host to - /// enumerate upstreams (e.g. to pre-warm a connection pool per deployment). - pub fn deployments(&self) -> &[Deployment] { - &self.model_list - } - - /// Whether any deployment is registered under `model`. - pub fn has_deployment(&self, model: &str) -> bool { - self.model_list - .iter() - .any(|deployment| deployment.model_name == model) - } - - /// Pick a deployment for `model` per the routing strategy. Returns `None` - /// when no deployment is registered under that `model_name`. - pub fn get_available_deployment(&self, model: &str) -> Option<&Deployment> { - let candidates: Vec<&Deployment> = self - .model_list - .iter() - .filter(|deployment| deployment.model_name == model) - .collect(); - self.routing_strategy.select(&candidates) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn deployment(name: &str, model: &str) -> Deployment { - Deployment { - model_name: name.to_string(), - litellm_params: LiteLLMParams { - model: model.to_string(), - api_key: None, - api_base: None, - }, - } - } - - #[test] - fn selects_a_matching_deployment() { - let router = Router::new(vec![ - deployment("gpt-realtime", "gpt-realtime"), - deployment("other", "other-model"), - ]); - let chosen = router - .get_available_deployment("gpt-realtime") - .expect("a deployment should match"); - assert_eq!(chosen.model_name, "gpt-realtime"); - } - - #[test] - fn unknown_model_returns_none() { - let router = Router::new(vec![deployment("gpt-realtime", "gpt-realtime")]); - assert!(router.get_available_deployment("missing").is_none()); - } -} diff --git a/litellm-rust/crates/core/src/router/strategy/mod.rs b/litellm-rust/crates/core/src/router/strategy/mod.rs deleted file mode 100644 index 7e8ac217db3..00000000000 --- a/litellm-rust/crates/core/src/router/strategy/mod.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! Routing policy: how the router picks one deployment from a model group. -//! -//! One module per strategy; [`RoutingStrategy::select`] dispatches to it. New -//! strategies (least-busy, latency-based, …) get their own file here. - -mod simple_shuffle; - -use super::Deployment; - -/// How the router chooses among the deployments sharing a `model_name`. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum RoutingStrategy { - /// Uniform random pick among the matching deployments. - #[default] - SimpleShuffle, -} - -impl RoutingStrategy { - /// Choose one deployment from `candidates` (all sharing the requested - /// `model_name`). Returns `None` when there are no candidates. - pub fn select<'a>(&self, candidates: &[&'a Deployment]) -> Option<&'a Deployment> { - match self { - RoutingStrategy::SimpleShuffle => simple_shuffle::select(candidates), - } - } -} diff --git a/litellm-rust/crates/core/src/router/strategy/simple_shuffle.rs b/litellm-rust/crates/core/src/router/strategy/simple_shuffle.rs deleted file mode 100644 index 74ce0c21e80..00000000000 --- a/litellm-rust/crates/core/src/router/strategy/simple_shuffle.rs +++ /dev/null @@ -1,47 +0,0 @@ -//! `simple-shuffle`: a uniform random pick among the candidate deployments. - -use rand::seq::SliceRandom; - -use crate::router::Deployment; - -/// Uniform random choice among `candidates` (all sharing the requested -/// `model_name`). Returns `None` when there are no candidates. -pub fn select<'a>(candidates: &[&'a Deployment]) -> Option<&'a Deployment> { - candidates.choose(&mut rand::thread_rng()).copied() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::router::{Deployment, LiteLLMParams}; - - fn deployment(model: &str) -> Deployment { - Deployment { - model_name: "gpt-realtime".to_string(), - litellm_params: LiteLLMParams { - model: model.to_string(), - api_key: None, - api_base: None, - }, - } - } - - #[test] - fn picks_from_candidates() { - let a = deployment("key-a"); - let b = deployment("key-b"); - let candidates = vec![&a, &b]; - for _ in 0..20 { - let chosen = select(&candidates).expect("non-empty"); - assert!(matches!( - chosen.litellm_params.model.as_str(), - "key-a" | "key-b" - )); - } - } - - #[test] - fn empty_candidates_select_none() { - assert!(select(&[]).is_none()); - } -} diff --git a/litellm-rust/crates/core/src/routing_utils/README.md b/litellm-rust/crates/core/src/routing_utils/README.md deleted file mode 100644 index 8585c18e421..00000000000 --- a/litellm-rust/crates/core/src/routing_utils/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Routing Utils - -Shared helpers for deciding how a LiteLLM model routes to an LLM provider. -Keep provider-name parsing, explicit `custom_llm_provider` handling, and model-prefix normalization here. -Do not put deployment selection or load-balancing logic here; that belongs in `router`. -Do not put provider HTTP transformation logic here; that belongs in `providers`. -Helpers in this folder should be deterministic and easy to unit test without network calls. diff --git a/litellm-rust/crates/core/src/routing_utils/mod.rs b/litellm-rust/crates/core/src/routing_utils/mod.rs deleted file mode 100644 index 8336397f870..00000000000 --- a/litellm-rust/crates/core/src/routing_utils/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod provider; diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 42fad740870..c112502b76f 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -14,14 +14,9 @@ default = ["abi3"] abi3 = ["pyo3/abi3-py310"] extension-module = ["pyo3/extension-module"] panic-test = [] -trace-parity = [ - "dep:tracing", - "litellm-core/observability", -] [dependencies] futures-util.workspace = true -tracing = { workspace = true, optional = true } litellm-core = { workspace = true, features = ["bedrock-auth"] } litellm-token-counter.workspace = true litellm-python-interop.workspace = true @@ -35,7 +30,6 @@ tokio = { workspace = true, features = ["sync"] } criterion.workspace = true rstest.workspace = true tokio-tungstenite.workspace = true -tracing.workspace = true [[bench]] name = "serialization" diff --git a/litellm-rust/crates/python-bridge/src/function_trace.rs b/litellm-rust/crates/python-bridge/src/function_trace.rs deleted file mode 100644 index bc3c962f7a3..00000000000 --- a/litellm-rust/crates/python-bridge/src/function_trace.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::fmt::Display; -use std::future::Future; - -use litellm_core::observability::{FunctionTrace, FunctionTraceEvent}; -use serde::Serialize; -use tracing::instrument::WithSubscriber; - -#[derive(Serialize)] -pub(crate) struct TracedResponse { - #[serde(skip_serializing_if = "Option::is_none")] - response: Option, - #[serde(skip_serializing_if = "Option::is_none")] - error: Option, - trace: Vec, -} - -pub(crate) async fn capture( - future: impl Future>, -) -> Result, E> -where - E: Display, -{ - let trace = FunctionTrace::default(); - let result = future.with_subscriber(trace.dispatcher()).await; - let events = trace.events(); - Ok(match result { - Ok(response) => TracedResponse { - response: Some(response), - error: None, - trace: events, - }, - Err(error) => TracedResponse { - response: None, - error: Some(error.to_string()), - trace: events, - }, - }) -} diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index 12bc57a8931..27ea2c1913e 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -3,8 +3,6 @@ mod constants; mod diagnostics; mod errors; mod execution; -#[cfg(feature = "trace-parity")] -mod function_trace; mod lifecycle; mod marshal; mod routes; @@ -124,39 +122,6 @@ mod tests { .filter(|name| !name.starts_with('_')) .collect(); assert_eq!(public_names, expected); - - #[cfg(not(feature = "trace-parity"))] - assert!(!module.hasattr("_trace").expect("module lookup should work")); - - #[cfg(feature = "trace-parity")] - { - let trace = module - .getattr("_trace") - .expect("trace build should expose its diagnostic namespace"); - let trace_names: Vec = trace - .cast::() - .expect("trace namespace should be a module") - .dict() - .keys() - .extract::>() - .expect("trace names should be strings") - .into_iter() - .filter(|name| !name.starts_with("__")) - .collect(); - assert_eq!( - trace_names, - [ - "ocr", - "aocr", - "transcription", - "atranscription", - "messages", - "amessages", - "chat_completions", - "achat_completions", - ] - ); - } }); } diff --git a/litellm-rust/crates/python-bridge/src/routes/audio_transcription/mod.rs b/litellm-rust/crates/python-bridge/src/routes/audio_transcription/mod.rs index f2997ee278c..68b701802a9 100644 --- a/litellm-rust/crates/python-bridge/src/routes/audio_transcription/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/audio_transcription/mod.rs @@ -5,8 +5,3 @@ use pyo3::prelude::*; pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { value::register(module) } - -#[cfg(feature = "trace-parity")] -pub(super) fn register_trace(module: &Bound<'_, PyModule>) -> PyResult<()> { - value::register_trace(module) -} diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs index f2997ee278c..68b701802a9 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs @@ -5,8 +5,3 @@ use pyo3::prelude::*; pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { value::register(module) } - -#[cfg(feature = "trace-parity")] -pub(super) fn register_trace(module: &Bound<'_, PyModule>) -> PyResult<()> { - value::register_trace(module) -} diff --git a/litellm-rust/crates/python-bridge/src/routes/definition.rs b/litellm-rust/crates/python-bridge/src/routes/definition.rs index 571042062f5..d7d868fd1f4 100644 --- a/litellm-rust/crates/python-bridge/src/routes/definition.rs +++ b/litellm-rust/crates/python-bridge/src/routes/definition.rs @@ -58,70 +58,6 @@ macro_rules! bridge_route { Ok(()) } - #[cfg(feature = "trace-parity")] - mod trace { - use pyo3::prelude::*; - use super::{$inputs, $map_error, $prepare}; - - #[pyfunction] - #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))] - #[allow(clippy::too_many_arguments)] - fn $sync_name( - py: pyo3::Python<'_>, - $($(#[$required_attr])* $required_name: $required_type,)* - $($(#[$optional_attr])* $optional_name: $optional_type,)* - ) -> pyo3::PyResult> { - let future = $prepare($inputs { - $($required_name,)* - $($optional_name),* - })?; - $crate::execution::run_sync( - py, - $crate::function_trace::capture(future), - $map_error, - ) - } - - #[pyfunction] - #[pyo3(signature = ($($required_name),*, $($optional_name=None),*))] - #[allow(clippy::too_many_arguments)] - fn $async_name( - py: pyo3::Python<'_>, - $($(#[$required_attr])* $required_name: $required_type,)* - $($(#[$optional_attr])* $optional_name: $optional_type,)* - ) -> pyo3::PyResult> { - let future = $prepare($inputs { - $($required_name,)* - $($optional_name),* - })?; - $crate::execution::run_async( - py, - $crate::function_trace::capture(future), - $map_error, - ) - } - - pub(super) fn register( - module: &pyo3::Bound<'_, pyo3::types::PyModule>, - ) -> pyo3::PyResult<()> { - $crate::routes::definition::add_function( - module, - pyo3::wrap_pyfunction!($sync_name, module)?, - )?; - $crate::routes::definition::add_function( - module, - pyo3::wrap_pyfunction!($async_name, module)?, - )?; - Ok(()) - } - } - - #[cfg(feature = "trace-parity")] - pub(super) fn register_trace( - module: &pyo3::Bound<'_, pyo3::types::PyModule>, - ) -> pyo3::PyResult<()> { - trace::register(module) - } }; } @@ -188,7 +124,6 @@ mod tests { Ok(execute_echo(inputs, drop_guard)) } - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] async fn execute_echo( inputs: EchoInputs, drop_guard: Option, @@ -548,33 +483,6 @@ asyncio.run(exercise()) }); } - #[cfg(feature = "trace-parity")] - #[test] - fn diagnostic_route_returns_the_response_and_filtered_trace() { - Python::initialize(); - Python::attach(|py| { - let module = PyModule::new(py, "synthetic").expect("module should be created"); - synthetic::register_trace(&module).expect("trace routes should register"); - let locals = PyDict::new(py); - locals - .set_item("routes", &module) - .expect("module should enter Python locals"); - let code = CString::new( - r#" -result = routes.echo("traced") -assert result["response"] == "traced", result -assert [event["function"] for event in result["trace"]] == ["execute_echo"], result -failure = routes.echo("error") -assert failure["error"] == "invalid request: synthetic error", failure -assert [event["function"] for event in failure["trace"]] == ["execute_echo"], failure -"#, - ) - .expect("Python source should not contain null bytes"); - py.run(&code, Some(&locals), Some(&locals)) - .expect("diagnostic route should return its response and trace"); - }); - } - #[test] fn route_registration_rejects_duplicate_python_names() { Python::initialize(); diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs index f2997ee278c..68b701802a9 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs @@ -5,8 +5,3 @@ use pyo3::prelude::*; pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { value::register(module) } - -#[cfg(feature = "trace-parity")] -pub(super) fn register_trace(module: &Bound<'_, PyModule>) -> PyResult<()> { - value::register_trace(module) -} diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index 97c39a5d6b3..4e2530a94f8 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -13,15 +13,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { audio_transcription::register(module)?; messages::register(module)?; chat_completions::register(module)?; - - #[cfg(feature = "trace-parity")] - { - let trace = PyModule::new(module.py(), "_trace")?; - ocr::register_trace(&trace)?; - audio_transcription::register_trace(&trace)?; - messages::register_trace(&trace)?; - chat_completions::register_trace(&trace)?; - module.add_submodule(&trace)?; - } Ok(()) } diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index 10fa40b65ea..f17bf249b7f 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -12,8 +12,3 @@ pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { document::register(module)?; lifecycle::register(module) } - -#[cfg(feature = "trace-parity")] -pub(super) fn register_trace(module: &Bound<'_, PyModule>) -> PyResult<()> { - value::register_trace(module) -} diff --git a/tests/rust-python-harness/AGENTS.md b/tests/rust-python-harness/AGENTS.md index 71e17541fd2..b66eaaeda9b 100644 --- a/tests/rust-python-harness/AGENTS.md +++ b/tests/rust-python-harness/AGENTS.md @@ -25,17 +25,6 @@ tests/rust-python-harness/ │ │ ├── ocr/ │ │ └── transcription/ │ │ -│ ├── unit_tests_mapping/ -│ │ ├── __init__.py -│ │ ├── contracts.py -│ │ ├── cases/ -│ │ │ └── ocr.py -│ │ ├── mapping_report.py -│ │ ├── mappings.py -│ │ ├── mapping_validator.py -│ │ ├── reporting.py -│ │ └── runner.py -│ │ │ ├── unit_tests_parity/ │ │ ├── __init__.py │ │ ├── reporting.py @@ -52,6 +41,7 @@ tests/rust-python-harness/ ├── reporting/ │ └── strategy.py └── unit_runners/ + ├── contracts.py └── suite_runner.py ``` @@ -63,10 +53,9 @@ tests/rust-python-harness/ - Examples: `run e2e_parity --surface sdk --function ocr`, `run unit_tests_parity --function ocr --pytest-arg=-x`, or `run all --function ocr` - `cli/catalog.py` discovers strategies, validates their Python definitions, and orders them; `cli/__init__.py` builds the Click command tree; `cli/commands.py` runs selected cases - `e2e_parity/` compares SDK objects, exceptions, callbacks, and streams, or gateway HTTP responses -- `trace_parity/` prints every collected Python call under `litellm/` and every Rust span without comparing them; mappings only filter the separate unit-test mapping strategy. Before running it rebuilds the native bridge with the `trace-parity` feature whenever `litellm-rust` sources are newer than the installed extension (`shared/native_build.py`) +- `trace_parity/` profiles the Python call stack and prints every collected Python call under `litellm/`; it never collects Rust spans and never rebuilds the native extension - E2E and trace strategies load their registered module cases and run surface-specific execution from their folders -- `unit_tests_mapping/contracts.py` owns typed harness-side mapping contracts, per-function contracts live below `cases/`, and `mappings.py` exports the registry; live test discovery derives unmapped Python and Rust-only tests without an exhaustive manifest -- `unit_tests_mapping/runner.py` validates confirmed mappings against the live Python and Rust inventories and attaches the derived status report +- `shared/unit_runners/contracts.py` owns the typed per-function unit contracts consumed by `unit_tests_parity` and `unit_tests_rust` - `unit_tests_parity/runner.py` runs each contract's `unit_parity_scope` with `LITELLM_RUST=0` and `LITELLM_RUST=1` in separate processes and requires matching outcomes, including failures; exclusions require a reason in the contract - `unit_tests_rust/runner.py` runs each contract's focused Cargo test suite; native Rust unit tests stay beside their implementation - `shared/unit_runners/suite_runner.py` runs typed suites registered in code with nodeids of the form `suite:::` @@ -74,4 +63,4 @@ tests/rust-python-harness/ - `shared/` contains reusable parity, tracing, reporting primitives, and unit-runner machinery - Keep fixtures with their owning API and existing Python tests in their current locations - Each strategy folder carries an `AGENTS.md` one-liner stating what it should be doing -- Run the harness's own checks with `uv run pytest -o consider_namespace_packages=true tests/rust-python-harness/shared tests/rust-python-harness/cli tests/rust-python-harness/strategies/unit_tests_mapping tests/rust-python-harness/strategies/unit_tests_parity tests/rust-python-harness/strategies/unit_tests_rust tests/test_rust_python_harness.py -q` +- Run the harness's own checks with `uv run pytest -o consider_namespace_packages=true tests/rust-python-harness/shared tests/rust-python-harness/cli tests/rust-python-harness/strategies/trace_parity tests/rust-python-harness/strategies/unit_tests_parity tests/rust-python-harness/strategies/unit_tests_rust tests/test_rust_python_harness.py -q` diff --git a/tests/rust-python-harness/cli/__init__.py b/tests/rust-python-harness/cli/__init__.py index d2bfdc55b19..13b995825dd 100644 --- a/tests/rust-python-harness/cli/__init__.py +++ b/tests/rust-python-harness/cli/__init__.py @@ -58,29 +58,16 @@ def _strategy_command(strategy: Strategy) -> click.Command: help=runner_argument.help, ) ) - for runner_option in strategy.definition.runner_options: - name: Final = runner_option.option.removeprefix("--").replace("-", "_") - params.append( - click.Option( - (runner_option.option, name), - type=click.Choice(runner_option.choices), - help=runner_option.help, - ) - ) def run_strategy( sdk_functions: tuple[str, ...], surface: str | None = None, runner_args: tuple[str, ...] = (), - **runner_options: str | None, ) -> int: selected_functions: Final = cast(frozenset[SdkFunction], frozenset(sdk_functions)) selected_surface: Final = cast(Surface | None, surface) cases: Final = select_cases((strategy,), selected_functions, selected_surface) - option_args: Final = tuple( - f"--{name.replace('_', '-')}={value}" for name, value in runner_options.items() if value is not None - ) - return run_command((strategy,), cases, (*runner_args, *option_args)) + return run_command((strategy,), cases, runner_args) return click.Command( strategy.id, diff --git a/tests/rust-python-harness/cli/test_cli.py b/tests/rust-python-harness/cli/test_cli.py index 5641aa8a539..219e1b0c6b7 100644 --- a/tests/rust-python-harness/cli/test_cli.py +++ b/tests/rust-python-harness/cli/test_cli.py @@ -19,7 +19,6 @@ from ..shared.reporting.models import ( ) from ..shared.reporting.strategy import NotImplementedCaseSpec, SkippedCaseSpec, StrategyDefinition from ..shared.reporting.ui import PlainDashboard, final_report, make_dashboard -from ..strategies.unit_tests_mapping.mappings import UNIT_TEST_CONTRACTS from ..strategies.unit_tests_parity import UNIT_PARITY_SUITES from ..strategies.unit_tests_rust import RUST_SUITES from . import main @@ -91,7 +90,6 @@ def test_should_load_surface_aware_and_function_only_strategies() -> None: assert [strategy.id for strategy in strategies] == [ "e2e_parity", "trace_parity", - "unit_tests_mapping", "unit_tests_parity", "unit_tests_rust", ] @@ -104,29 +102,23 @@ def test_should_load_surface_aware_and_function_only_strategies() -> None: def test_unit_strategies_use_function_only_cases() -> None: strategies: Final = { - strategy.id: strategy - for strategy in load_catalog() - if strategy.id in {"unit_tests_mapping", "unit_tests_parity", "unit_tests_rust"} + strategy.id: strategy for strategy in load_catalog() if strategy.id in {"unit_tests_parity", "unit_tests_rust"} } for sdk_function in SDK_FUNCTIONS: cases: Final = tuple( case for strategy in strategies.values() for case in strategy.cases if case.sdk_function == sdk_function ) - assert len(cases) == 3 + assert len(cases) == 2 assert all(case.surface is None for case in cases) - expected_mapping: Final = ( - CaseDisposition.RUNNABLE if sdk_function in UNIT_TEST_CONTRACTS else CaseDisposition.NOT_IMPLEMENTED - ) - assert cases[0].spec.disposition is expected_mapping expected_parity: Final = ( CaseDisposition.RUNNABLE if sdk_function in UNIT_PARITY_SUITES else CaseDisposition.NOT_IMPLEMENTED ) expected_rust: Final = ( CaseDisposition.RUNNABLE if sdk_function in RUST_SUITES else CaseDisposition.NOT_IMPLEMENTED ) - assert cases[1].spec.disposition is expected_parity - assert cases[2].spec.disposition is expected_rust + assert cases[0].spec.disposition is expected_parity + assert cases[1].spec.disposition is expected_rust def test_raw_dashboard_is_always_the_default() -> None: @@ -243,7 +235,6 @@ def test_every_unavailable_case_finishes_and_explains_itself() -> None: section_titles: Final = { "e2e_parity": "End-to-end parity outcomes", "trace_parity": "traces", - "unit_tests_mapping": "Python/Rust unit-test mappings", "unit_tests_parity": "Python backend parity outcomes", "unit_tests_rust": "Native Rust unit-test outcomes", } @@ -264,7 +255,6 @@ def test_every_unavailable_case_finishes_and_explains_itself() -> None: ("e2e_parity", "--surface", "--pytest-arg"), ("trace_parity", "--surface", "--pytest-arg"), ("unit_tests_parity", "--pytest-arg", "--surface"), - ("unit_tests_mapping", "--detail", "--surface"), ("unit_tests_rust", "--function", "--surface"), ), ) @@ -291,7 +281,6 @@ def test_run_help_lists_all_and_every_strategy(capsys: pytest.CaptureFixture[str "all", "e2e_parity", "trace_parity", - "unit_tests_mapping", "unit_tests_parity", "unit_tests_rust", ): @@ -359,7 +348,7 @@ def test_strategy_command_forwards_repeated_filters_and_runner_arguments( ] -def test_trace_command_forwards_engine_and_scenario(monkeypatch: pytest.MonkeyPatch) -> None: +def test_trace_command_forwards_scenario(monkeypatch: pytest.MonkeyPatch) -> None: cli: Final = importlib.import_module("tests.rust-python-harness.cli") captured: list[tuple[str, ...]] = [] @@ -374,8 +363,8 @@ def test_trace_command_forwards_engine_and_scenario(monkeypatch: pytest.MonkeyPa monkeypatch.setattr(cli, "run_command", capture_run) - assert main(["run", "trace_parity", "--scenario", "async-mistral", "--engine", "python"]) == 0 - assert captured == [("async-mistral", "--engine=python")] + assert main(["run", "trace_parity", "--scenario", "async-mistral"]) == 0 + assert captured == [("async-mistral",)] def test_omitted_surface_selects_every_strategy_surface(monkeypatch: pytest.MonkeyPatch) -> None: @@ -413,8 +402,8 @@ def test_run_all_selects_every_declared_case_once(monkeypatch: pytest.MonkeyPatc monkeypatch.setattr(cli, "run_command", capture_run) assert main(["run", "all", "--function", "ocr"]) == 0 - assert len(selected) == 7 - assert sum(case.surface is None for case in selected) == 3 + assert len(selected) == 6 + assert sum(case.surface is None for case in selected) == 2 assert sum(case.surface is not None for case in selected) == 4 diff --git a/tests/rust-python-harness/shared/native_build.py b/tests/rust-python-harness/shared/native_build.py deleted file mode 100644 index f67488cecb4..00000000000 --- a/tests/rust-python-harness/shared/native_build.py +++ /dev/null @@ -1,115 +0,0 @@ -from __future__ import annotations - -import importlib.util -import os -import subprocess -import sys -from collections.abc import Iterator -from pathlib import Path -from typing import Final - -from litellm.rust_bridge import get_native_bridge, reset_native_bridge_cache - -MATURIN_SPEC: Final = "maturin==1.15.0" -BRIDGE_FEATURE: Final = "trace-parity" -_RUST_ROOT: Final = "litellm-rust" -_LOCKFILE: Final = "Cargo.lock" -_SOURCE_SUFFIXES: Final = frozenset({".rs", ".toml"}) -_FAILURE_OUTPUT_LINES: Final = 15 -_TRACE_CHECK: Final = ( - "from litellm.rust_bridge import get_native_bridge; " - "bridge = get_native_bridge(); " - "raise SystemExit(0 if bridge is not None and getattr(bridge, '_trace', None) is not None else 1)" -) - - -def needs_rebuild(native_mtime: float | None, newest_source_mtime: float | None) -> bool: - if native_mtime is None: - return True - if newest_source_mtime is None: - return False - return newest_source_mtime > native_mtime - - -def _source_files(rust_root: Path) -> Iterator[Path]: - for path in rust_root.rglob("*"): - relative: Final = path.relative_to(rust_root) - if "target" in relative.parts or not path.is_file(): - continue - if path.name == _LOCKFILE or path.suffix in _SOURCE_SUFFIXES: - yield path - - -def _newest_source_mtime(repo_root: Path) -> float | None: - rust_root: Final = repo_root / _RUST_ROOT - if not rust_root.is_dir(): - return None - return max((path.stat().st_mtime for path in _source_files(rust_root)), default=None) - - -def _native_module_path() -> Path | None: - try: - spec: Final = importlib.util.find_spec("litellm.rust_bridge._native") - except (ImportError, ValueError): - return None - origin: Final = getattr(spec, "origin", None) - return Path(origin) if origin else None - - -def _drop_imported_bridge() -> None: - reset_native_bridge_cache() - for name in tuple(sys.modules): - if name.startswith("litellm.rust_bridge._native"): - del sys.modules[name] - - -def _rebuild(repo_root: Path) -> tuple[bool, str]: - command: Final = ("uvx", "--from", MATURIN_SPEC, "maturin", "develop", "--features", BRIDGE_FEATURE) - completed: Final = subprocess.run( - command, - cwd=repo_root, - env={**os.environ, "VIRTUAL_ENV": sys.prefix}, - capture_output=True, - text=True, - check=False, - ) - output: Final = f"{completed.stdout}\n{completed.stderr}".strip() - lines: Final = tuple(output.splitlines()) - return completed.returncode == 0, "\n".join(lines[-_FAILURE_OUTPUT_LINES:]) - - -def _installed_bridge_has_trace(repo_root: Path) -> bool: - completed: Final = subprocess.run( - (sys.executable, "-c", _TRACE_CHECK), - cwd=repo_root, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - ) - return completed.returncode == 0 - - -def trace_bridge_error() -> str | None: - bridge: Final = get_native_bridge() - if bridge is None: - return "native Rust bridge is not importable" - if getattr(bridge, "_trace", None) is None: - return f"native Rust bridge does not expose _trace; it must be built with the {BRIDGE_FEATURE} feature" - return None - - -def ensure_trace_bridge(repo_root: Path) -> str | None: - native_path: Final = _native_module_path() - native_mtime: Final = native_path.stat().st_mtime if native_path is not None and native_path.exists() else None - rebuild_required: Final = needs_rebuild( - native_mtime, _newest_source_mtime(repo_root) - ) or not _installed_bridge_has_trace(repo_root) - if rebuild_required: - print(f"Rebuilding native Rust bridge ({BRIDGE_FEATURE} feature)...", flush=True) - succeeded: Final - output: Final - succeeded, output = _rebuild(repo_root) - if not succeeded: - return f"native Rust bridge rebuild failed:\n{output}" - _drop_imported_bridge() - return trace_bridge_error() diff --git a/tests/rust-python-harness/shared/reporting/strategy.py b/tests/rust-python-harness/shared/reporting/strategy.py index d8e9d9e5ba9..7e76f035e20 100644 --- a/tests/rust-python-harness/shared/reporting/strategy.py +++ b/tests/rust-python-harness/shared/reporting/strategy.py @@ -67,13 +67,6 @@ class RunnerArgumentDefinition: metavar: str = "ARG" -@dataclass(frozen=True, slots=True) -class RunnerOptionDefinition: - option: str - help: str - choices: tuple[str, ...] - - class StrategyRunner(Protocol): def __call__( self, @@ -97,4 +90,3 @@ class StrategyDefinition: render: StrategyRenderer surfaces: tuple[Surface, ...] = () runner_argument: RunnerArgumentDefinition | None = None - runner_options: tuple[RunnerOptionDefinition, ...] = () diff --git a/tests/rust-python-harness/shared/test_native_build.py b/tests/rust-python-harness/shared/test_native_build.py deleted file mode 100644 index dc08bc1a2b6..00000000000 --- a/tests/rust-python-harness/shared/test_native_build.py +++ /dev/null @@ -1,121 +0,0 @@ -from __future__ import annotations - -import os -from types import SimpleNamespace -from typing import Final - -import pytest - -from . import native_build - - -def test_needs_rebuild_when_bridge_is_missing() -> None: - assert native_build.needs_rebuild(None, 1.0) - - -def test_needs_rebuild_when_sources_are_newer_than_bridge() -> None: - assert native_build.needs_rebuild(1.0, 2.0) - - -def test_fresh_bridge_with_older_sources_needs_no_rebuild() -> None: - assert not native_build.needs_rebuild(2.0, 1.0) - - -def test_bridge_without_rust_sources_needs_no_rebuild() -> None: - assert not native_build.needs_rebuild(2.0, None) - - -def test_newest_source_mtime_tracks_rust_sources_and_skips_target(tmp_path: Final) -> None: - source: Final = tmp_path / "litellm-rust" / "crates" / "bridge" / "src" - source.mkdir(parents=True) - (source / "lib.rs").write_text("fn main() {}\n") - os.utime(source / "lib.rs", (1_000, 1_000)) - manifest: Final = tmp_path / "litellm-rust" / "crates" / "bridge" / "Cargo.toml" - manifest.write_text("[package]\n") - os.utime(manifest, (2_000, 2_000)) - lockfile: Final = tmp_path / "litellm-rust" / "Cargo.lock" - lockfile.write_text("") - os.utime(lockfile, (1_500, 1_500)) - target: Final = tmp_path / "litellm-rust" / "target" / "debug" / "junk.rs" - target.parent.mkdir(parents=True) - target.write_text("fn main() {}\n") - os.utime(target, (9_999, 9_999)) - - assert native_build._newest_source_mtime(tmp_path) == 2_000.0 - - -def test_newest_source_mtime_is_none_without_rust_workspace(tmp_path: Final) -> None: - assert native_build._newest_source_mtime(tmp_path) is None - - -def test_ensure_trace_bridge_rebuilds_when_stale( - tmp_path: Final, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - native: Final = tmp_path / "_native.abi3.so" - native.write_bytes(b"") - os.utime(native, (1_000, 1_000)) - source: Final = tmp_path / "litellm-rust" / "crates" / "bridge" / "src" / "lib.rs" - source.parent.mkdir(parents=True) - source.write_text("fn main() {}\n") - os.utime(source, (2_000, 2_000)) - state: Final = SimpleNamespace(rebuilt=False) - - def fake_rebuild(repo_root: object) -> tuple[bool, str]: - state.rebuilt = True - return True, "" - - monkeypatch.setattr(native_build, "_native_module_path", lambda: native) - monkeypatch.setattr(native_build, "_rebuild", fake_rebuild) - monkeypatch.setattr(native_build, "_drop_imported_bridge", lambda: None) - monkeypatch.setattr(native_build, "get_native_bridge", lambda: SimpleNamespace(_trace=object())) - - assert native_build.ensure_trace_bridge(tmp_path) is None - assert state.rebuilt is True - assert "Rebuilding native Rust bridge" in capsys.readouterr().out - - -def test_ensure_trace_bridge_reports_failed_rebuild(tmp_path: Final, monkeypatch: pytest.MonkeyPatch) -> None: - source: Final = tmp_path / "litellm-rust" / "crates" / "bridge" / "src" / "lib.rs" - source.parent.mkdir(parents=True) - source.write_text("fn main() {}\n") - - monkeypatch.setattr(native_build, "_native_module_path", lambda: None) - monkeypatch.setattr(native_build, "_rebuild", lambda repo_root: (False, "boom")) - - message: Final = native_build.ensure_trace_bridge(tmp_path) - - assert message is not None - assert "rebuild failed" in message - assert "boom" in message - - -def test_ensure_trace_bridge_rebuilds_when_trace_feature_is_missing( - tmp_path: Final, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - native: Final = tmp_path / "_native.abi3.so" - native.write_bytes(b"") - os.utime(native, (9_999, 9_999)) - source: Final = tmp_path / "litellm-rust" / "crates" / "bridge" / "src" / "lib.rs" - source.parent.mkdir(parents=True) - source.write_text("fn main() {}\n") - os.utime(source, (1_000, 1_000)) - state: Final = SimpleNamespace(rebuilt=False) - - def fake_rebuild(repo_root: object) -> tuple[bool, str]: - state.rebuilt = True - return True, "" - - def fake_get_native_bridge() -> SimpleNamespace: - assert state.rebuilt - return SimpleNamespace(_trace=object()) - - monkeypatch.setattr(native_build, "_native_module_path", lambda: native) - monkeypatch.setattr(native_build, "_rebuild", fake_rebuild) - monkeypatch.setattr(native_build, "_installed_bridge_has_trace", lambda repo_root: False) - monkeypatch.setattr(native_build, "get_native_bridge", fake_get_native_bridge) - - message: Final = native_build.ensure_trace_bridge(tmp_path) - - assert message is None - assert state.rebuilt is True - assert "Rebuilding native Rust bridge" in capsys.readouterr().out diff --git a/tests/rust-python-harness/shared/tracing/native.py b/tests/rust-python-harness/shared/tracing/native.py deleted file mode 100644 index 688995cbc4b..00000000000 --- a/tests/rust-python-harness/shared/tracing/native.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import annotations - -from typing import Final - -from pydantic import BaseModel, ConfigDict - -from .profiler import FunctionTraceEvent - - -class _TraceEventPayload(BaseModel): - model_config = ConfigDict(strict=True, extra="forbid") - id: int - parent_id: int | None - function: str - module_path: str | None = None - file: str | None = None - line: int | None = None - - -class TraceResponsePayload(BaseModel): - model_config = ConfigDict(strict=True, extra="forbid") - response: object = None - error: str | None = None - trace: tuple[_TraceEventPayload, ...] | list[_TraceEventPayload] - - -def native_trace_events(payload: object) -> tuple[FunctionTraceEvent, ...]: - response: Final = TraceResponsePayload.model_validate(payload) - return tuple( - FunctionTraceEvent( - event.id, - event.parent_id, - event.function, - event.module_path, - event.file, - event.line, - ) - for event in response.trace - ) diff --git a/tests/rust-python-harness/shared/tracing/steps.py b/tests/rust-python-harness/shared/tracing/steps.py index 492ffab64e5..415e3f02efc 100644 --- a/tests/rust-python-harness/shared/tracing/steps.py +++ b/tests/rust-python-harness/shared/tracing/steps.py @@ -1,46 +1,10 @@ from __future__ import annotations -import re -from collections import Counter from collections.abc import Sequence from dataclasses import dataclass -from typing import Final, Literal from .profiler import FunctionTraceEvent -Engine = Literal["python", "rust"] - - -@dataclass(frozen=True, slots=True) -class TraceMapping: - span: str - python: re.Pattern[str] | None - rust: str | None - - -def mapping( - *, - python_frame: str | None = None, - rust_span: str | None = None, - span: str | None = None, -) -> TraceMapping: - if rust_span is None: - if python_frame is None: - raise ValueError("mapping needs a python_frame pattern, a rust_span name, or both") - if span is None: - raise ValueError("a python-only mapping needs an explicit span to compare under") - return TraceMapping(span, re.compile(python_frame), None) - if python_frame is None: - return TraceMapping(rust_span, None, rust_span) - if span is not None and span != rust_span: - raise ValueError(f"span {span!r} disagrees with rust_span {rust_span!r}") - return TraceMapping(rust_span, re.compile(python_frame), rust_span) - - -@dataclass(frozen=True, slots=True) -class TraceContract: - unordered_children_of: frozenset[str] = frozenset() - @dataclass(frozen=True, slots=True) class PipelineStep: @@ -50,61 +14,22 @@ class PipelineStep: raw: str -@dataclass(frozen=True, slots=True) -class PipelineProjection: - steps: tuple[PipelineStep, ...] = () - unmatched: int = 0 - - -def _span_for(engine: Engine, function: str, mappings: Sequence[TraceMapping]) -> str | None: - matches: Final = tuple( - item.span - for item in mappings - if ( - engine == "python" - and item.python is not None - and item.python.search(function) - or engine == "rust" - and item.rust == function - ) - ) - if len(matches) > 1: - raise ValueError(f"{engine} event {function!r} matches multiple trace mappings: {matches}") - if matches: - return matches[0] - return function if engine == "rust" else None - - -def pipeline_projection( - engine: Engine, events: Sequence[FunctionTraceEvent], mappings: Sequence[TraceMapping] | None = None -) -> PipelineProjection: +def pipeline_projection(events: Sequence[FunctionTraceEvent]) -> tuple[PipelineStep, ...]: raw_parents: dict[int, int | None] = {} projected_ids: set[int] = set() shown: list[PipelineStep] = [] - unmatched: int = 0 for event in events: if event.id in raw_parents: raise ValueError(f"duplicate trace event id {event.id}") if event.parent_id is not None and event.parent_id not in raw_parents: raise ValueError(f"trace event {event.id} references unknown or later parent {event.parent_id}") raw_parents[event.id] = event.parent_id - span = event.function if mappings is None else _span_for(engine, event.function, mappings) - if span is None: - unmatched += 1 - continue parent_id: int | None = event.parent_id while parent_id is not None and parent_id not in projected_ids: parent_id = raw_parents[parent_id] - shown.append(PipelineStep(event.id, parent_id, span, event.raw)) + shown.append(PipelineStep(event.id, parent_id, event.function, event.raw)) projected_ids.add(event.id) - return PipelineProjection(tuple(shown), unmatched) - - -@dataclass(frozen=True, slots=True) -class TraceNode: - id: int - span: str - children: tuple[TraceNode, ...] + return tuple(shown) def trace_depths(steps: Sequence[PipelineStep]) -> dict[int, int]: @@ -112,153 +37,3 @@ def trace_depths(steps: Sequence[PipelineStep]) -> dict[int, int]: for step in steps: depths[step.id] = 0 if step.parent_id is None else depths[step.parent_id] + 1 return depths - - -def _forest(steps: Sequence[PipelineStep]) -> tuple[TraceNode, ...]: - children: dict[int | None, list[PipelineStep]] = {} - known: set[int] = set() - for step in steps: - if step.id in known: - raise ValueError(f"duplicate projected event id {step.id}") - if step.parent_id is not None and step.parent_id not in known: - raise ValueError(f"projected event {step.id} references unknown or later parent {step.parent_id}") - known.add(step.id) - children.setdefault(step.parent_id, []).append(step) - - def node(step: PipelineStep) -> TraceNode: - return TraceNode(step.id, step.span, tuple(node(child) for child in children.get(step.id, ()))) - - return tuple(node(step) for step in children.get(None, ())) - - -def _exclusive_spans(engine: Engine, mappings: Sequence[TraceMapping]) -> frozenset[str]: - return frozenset( - item.span - for item in mappings - if (engine == "python" and item.rust is None) or (engine == "rust" and item.python is None) - ) - - -def _comparable_steps( - engine: Engine, steps: Sequence[PipelineStep], mappings: Sequence[TraceMapping] -) -> tuple[PipelineStep, ...]: - exclusive: Final = _exclusive_spans(engine, mappings) - raw_parents: Final = {step.id: step.parent_id for step in steps} - included: Final = {step.id for step in steps if step.span not in exclusive} - comparable: list[PipelineStep] = [] - for step in steps: - if step.id not in included: - continue - parent_id: int | None = step.parent_id - while parent_id is not None and parent_id not in included: - parent_id = raw_parents[parent_id] - comparable.append(PipelineStep(step.id, parent_id, step.span, step.raw)) - return tuple(comparable) - - -def _signature(node: TraceNode, contract: TraceContract) -> tuple[object, ...]: - children: tuple[tuple[object, ...], ...] = tuple(_signature(child, contract) for child in node.children) - normalized: Final = tuple(sorted(children, key=repr)) if node.span in contract.unordered_children_of else children - return (node.span, normalized) - - -def trace_signature( - engine: Engine, - steps: Sequence[PipelineStep], - mappings: Sequence[TraceMapping], - contract: TraceContract, -) -> tuple[tuple[object, ...], ...]: - return tuple(_signature(root, contract) for root in _forest(_comparable_steps(engine, steps, mappings))) - - -@dataclass(frozen=True, slots=True) -class TraceDiff: - python_only: tuple[str, ...] - rust_only: tuple[str, ...] - shared_order_matches: bool - missing_mappings: tuple[str, ...] = () - first_difference: str | None = None - - @property - def matches(self) -> bool: - return not self.python_only and not self.rust_only and not self.missing_mappings and self.shared_order_matches - - -def _missing_mappings( - python: Sequence[PipelineStep], rust: Sequence[PipelineStep], mappings: Sequence[TraceMapping] -) -> tuple[str, ...]: - python_seen: Final = frozenset(step.span for step in python) - rust_seen: Final = frozenset(step.span for step in rust) - return tuple( - item.span - for item in mappings - if (item.python is not None and item.span not in python_seen) - or (item.rust is not None and item.span not in rust_seen) - ) - - -def _first_difference( - python: Sequence[PipelineStep], - rust: Sequence[PipelineStep], - mappings: Sequence[TraceMapping], - contract: TraceContract, -) -> str | None: - python_forest: Final = _forest(_comparable_steps("python", python, mappings)) - rust_forest: Final = _forest(_comparable_steps("rust", rust, mappings)) - - def compare_children( - python_nodes: Sequence[TraceNode], rust_nodes: Sequence[TraceNode], path: str, *, unordered: bool - ) -> str | None: - if unordered: - python_signatures: Final = Counter(_signature(node, contract) for node in python_nodes) - rust_signatures: Final = Counter(_signature(node, contract) for node in rust_nodes) - if python_signatures != rust_signatures: - return f"{path}: unordered child subtree multiset differs" - return None - for index in range(max(len(python_nodes), len(rust_nodes))): - child_path = f"{path}/child[{index + 1}]" - if index >= len(python_nodes): - return f"{child_path}: Rust has extra {rust_nodes[index].span!r}" - if index >= len(rust_nodes): - return f"{child_path}: Python has extra {python_nodes[index].span!r}" - python_node = python_nodes[index] - rust_node = rust_nodes[index] - if python_node.span != rust_node.span: - return f"{child_path}: Python={python_node.span!r}, Rust={rust_node.span!r}" - difference = compare_children( - python_node.children, - rust_node.children, - f"{child_path}/{python_node.span}", - unordered=python_node.span in contract.unordered_children_of, - ) - if difference is not None: - return difference - return None - - return compare_children(python_forest, rust_forest, "root", unordered=False) - - -def trace_diff( - python: Sequence[PipelineStep], - rust: Sequence[PipelineStep], - mappings: Sequence[TraceMapping] = (), - contract: TraceContract = TraceContract(), -) -> TraceDiff: - python_comparable: Final = _comparable_steps("python", python, mappings) - rust_comparable: Final = _comparable_steps("rust", rust, mappings) - python_spans: Final = tuple(step.span for step in python_comparable) - rust_spans: Final = tuple(step.span for step in rust_comparable) - python_counts: Final = Counter(python_spans) - rust_counts: Final = Counter(rust_spans) - python_only_counts: Final = python_counts - rust_counts - rust_only_counts: Final = rust_counts - python_counts - python_only: Final = tuple(span for span, count in python_only_counts.items() for _ in range(count)) - rust_only: Final = tuple(span for span, count in rust_only_counts.items() for _ in range(count)) - first_difference: Final = _first_difference(python, rust, mappings, contract) - return TraceDiff( - python_only=python_only, - rust_only=rust_only, - shared_order_matches=bool(python_comparable or rust_comparable) and first_difference is None, - missing_mappings=_missing_mappings(python, rust, mappings), - first_difference=first_difference, - ) diff --git a/tests/rust-python-harness/shared/tracing/test_steps.py b/tests/rust-python-harness/shared/tracing/test_steps.py index ee5e0bafd28..cad9bf1aab5 100644 --- a/tests/rust-python-harness/shared/tracing/test_steps.py +++ b/tests/rust-python-harness/shared/tracing/test_steps.py @@ -5,43 +5,14 @@ from typing import Final import pytest from .profiler import FunctionTraceEvent -from .steps import Engine, TraceContract, mapping, pipeline_projection, trace_depths, trace_diff - -MAPPINGS: Final = ( - mapping(rust_span="route", python_frame=r"entry$"), - mapping(rust_span="provider", python_frame=r"provider$"), - mapping(rust_span="request", python_frame=r"request$"), - mapping(rust_span="http", python_frame=r"post$"), - mapping(rust_span="response", python_frame=r"response$"), -) +from .steps import pipeline_projection, trace_depths def event(event_id: int, function: str, parent_id: int | None = None) -> FunctionTraceEvent: return FunctionTraceEvent(event_id, parent_id, function) -def test_python_projection_collapses_unmapped_parents_and_counts_noise() -> None: - events: Final = ( - event(0, "module.py:1 entry"), - event(1, "noise", 0), - event(2, "module.py:2 provider", 1), - event(3, "module.py:3 request", 0), - event(4, "client.py:4 post", 3), - event(5, "module.py:5 response", 0), - ) - projection: Final = pipeline_projection("python", events, MAPPINGS) - assert projection.unmatched == 1 - assert [(step.id, step.parent_id, step.span, step.raw) for step in projection.steps] == [ - (0, None, "route", "module.py:1 entry"), - (2, 0, "provider", "module.py:2 provider"), - (3, 0, "request", "module.py:3 request"), - (4, 3, "http", "client.py:4 post"), - (5, 0, "response", "module.py:5 response"), - ] - - -@pytest.mark.parametrize("engine", ("python", "rust")) -def test_projection_without_mappings_keeps_every_call_and_parent(engine: Engine) -> None: +def test_projection_keeps_every_call_and_parent() -> None: events: Final = ( event(0, "module.py:1 entry"), event(1, "module.py:2 internal_helper", 0), @@ -49,124 +20,25 @@ def test_projection_without_mappings_keeps_every_call_and_parent(engine: Engine) event(3, "module.py:2 internal_helper", 0), ) - projection: Final = pipeline_projection(engine, events) + steps: Final = pipeline_projection(events) - assert projection.unmatched == 0 - assert tuple((step.id, step.parent_id, step.span, step.raw) for step in projection.steps) == tuple( + assert tuple((step.id, step.parent_id, step.span, step.raw) for step in steps) == tuple( (item.id, item.parent_id, item.function, item.raw) for item in events ) -def test_rust_projection_keeps_unknown_spans() -> None: - projection: Final = pipeline_projection("rust", (event(0, "route"), event(1, "new_span", 0)), MAPPINGS) - assert [(step.span, step.parent_id) for step in projection.steps] == [("route", None), ("new_span", 0)] - - def test_projection_preserves_repeated_occurrences() -> None: - projection: Final = pipeline_projection( - "rust", - (event(0, "route"), event(1, "http", 0), event(2, "http", 0)), - MAPPINGS, - ) - assert [step.span for step in projection.steps] == ["route", "http", "http"] + steps: Final = pipeline_projection((event(0, "route"), event(1, "http", 0), event(2, "http", 0))) + assert [step.span for step in steps] == ["route", "http", "http"] def test_projection_preserves_multiple_roots() -> None: - projection: Final = pipeline_projection("rust", (event(0, "route"), event(1, "request")), MAPPINGS) - assert trace_depths(projection.steps) == {0: 0, 1: 0} + steps: Final = pipeline_projection((event(0, "route"), event(1, "request"))) + assert trace_depths(steps) == {0: 0, 1: 0} def test_projection_rejects_duplicate_and_unknown_parent_ids() -> None: with pytest.raises(ValueError, match="duplicate trace event id"): - pipeline_projection("rust", (event(0, "route"), event(0, "request")), MAPPINGS) + pipeline_projection((event(0, "route"), event(0, "request"))) with pytest.raises(ValueError, match="unknown or later parent"): - pipeline_projection("rust", (event(1, "request", 0),), MAPPINGS) - - -@pytest.mark.parametrize("engine", ("python", "rust")) -def test_rust_only_mappings_do_not_swallow_python_frames(engine: Engine) -> None: - projection: Final = pipeline_projection( - engine, - (event(0, "anything"),), - (mapping(rust_span="rust_only_span"),), - ) - if engine == "python": - assert projection.unmatched == 1 - assert projection.steps == () - else: - assert projection.unmatched == 0 - assert projection.steps[0].span == "anything" - - -def test_mapping_builder_rejects_empty_and_ambiguous_declarations() -> None: - with pytest.raises(ValueError, match="mapping needs"): - mapping() - with pytest.raises(ValueError, match="python-only mapping needs"): - mapping(python_frame=r"frame$") - with pytest.raises(ValueError, match="disagrees with"): - mapping(rust_span="span_a", python_frame=r"frame$", span="span_b") - - -def test_projection_rejects_ambiguous_python_mapping() -> None: - mappings: Final = ( - mapping(rust_span="first", python_frame=r"same$"), - mapping(rust_span="second", python_frame=r"same$"), - ) - with pytest.raises(ValueError, match="multiple trace mappings"): - pipeline_projection("python", (event(0, "module.py:1 same"),), mappings) - - -def test_trace_diff_matches_identical_occurrence_trees() -> None: - mappings: Final = (MAPPINGS[0], MAPPINGS[2]) - steps: Final = pipeline_projection( - "rust", (event(0, "route"), event(1, "request", 0), event(2, "request", 0)), mappings - ).steps - assert trace_diff(steps, steps, mappings).matches - - -def test_trace_diff_rejects_missing_occurrence_and_parent_drift() -> None: - python: Final = pipeline_projection( - "rust", (event(0, "route"), event(1, "request", 0), event(2, "request", 0)), MAPPINGS - ).steps - missing: Final = pipeline_projection("rust", (event(0, "route"), event(1, "request", 0)), MAPPINGS).steps - reparented: Final = pipeline_projection( - "rust", (event(0, "route"), event(1, "request", 0), event(2, "request", 1)), MAPPINGS - ).steps - assert trace_diff(python, missing, MAPPINGS).python_only == ("request",) - assert not trace_diff(python, reparented, MAPPINGS).matches - - -def test_trace_diff_rejects_sequential_reorder() -> None: - first: Final = pipeline_projection( - "rust", (event(0, "route"), event(1, "request", 0), event(2, "response", 0)), MAPPINGS - ).steps - second: Final = pipeline_projection( - "rust", (event(0, "route"), event(1, "response", 0), event(2, "request", 0)), MAPPINGS - ).steps - diff: Final = trace_diff(first, second, MAPPINGS) - assert not diff.matches - assert diff.first_difference == "root/child[1]/route/child[1]: Python='request', Rust='response'" - - -def test_trace_diff_allows_reordered_concurrent_children() -> None: - mappings: Final = (MAPPINGS[0], MAPPINGS[2], MAPPINGS[4]) - first: Final = pipeline_projection( - "rust", (event(0, "route"), event(1, "request", 0), event(2, "response", 0)), mappings - ).steps - second: Final = pipeline_projection( - "rust", (event(0, "route"), event(1, "response", 0), event(2, "request", 0)), mappings - ).steps - contract: Final = TraceContract(frozenset({"route"})) - assert trace_diff(first, second, mappings, contract).matches - - -def test_trace_diff_prunes_declared_engine_only_nodes_but_requires_them() -> None: - mappings: Final = (MAPPINGS[0], mapping(rust_span="rust_prepare")) - python: Final = pipeline_projection("python", (event(0, "module.py:1 entry"),), mappings).steps - rust: Final = pipeline_projection("rust", (event(0, "route"), event(1, "rust_prepare", 0)), mappings).steps - assert trace_diff(python, rust, mappings).matches - assert trace_diff(python, rust[:1], mappings).missing_mappings == ("rust_prepare",) - - -def test_trace_diff_does_not_claim_empty_traces_match() -> None: - assert not trace_diff((), ()).matches + pipeline_projection((event(1, "request", 0),)) diff --git a/tests/rust-python-harness/shared/unit_runners/contracts.py b/tests/rust-python-harness/shared/unit_runners/contracts.py new file mode 100644 index 00000000000..e121ee515e6 --- /dev/null +++ b/tests/rust-python-harness/shared/unit_runners/contracts.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from collections import Counter +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator +from typing_extensions import Self + +from ..reporting.models import SdkFunction + + +class _ContractModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + +def _clean_unique(values: tuple[str, ...], field: str) -> tuple[str, ...]: + cleaned: Final = tuple(value.strip().rstrip("/") for value in values) + if not cleaned or any(not value for value in cleaned): + raise ValueError(f"{field} must contain non-empty paths") + duplicates: Final = tuple(value for value, count in Counter(cleaned).items() if count > 1) + if duplicates: + raise ValueError(f"{field} contains duplicates: {sorted(duplicates)}") + return cleaned + + +class UnitParityExclusionSpec(_ContractModel): + nodeid: str + reason: str + + @field_validator("nodeid", "reason") + @classmethod + def validate_fields(cls, value: str) -> str: + stripped: Final = value.strip() + if not stripped: + raise ValueError("must be a non-empty string") + return stripped + + +class UnitParitySpec(_ContractModel): + python_selectors: tuple[str, ...] + exclusions: tuple[UnitParityExclusionSpec, ...] = () + + @field_validator("python_selectors") + @classmethod + def validate_python_selectors(cls, value: tuple[str, ...]) -> tuple[str, ...]: + return _clean_unique(value, "unit parity python_selectors") + + @model_validator(mode="after") + def validate_exclusions(self) -> Self: + nodeids: Final = tuple(exclusion.nodeid for exclusion in self.exclusions) + duplicates: Final = tuple(nodeid for nodeid, count in Counter(nodeids).items() if count > 1) + if duplicates: + raise ValueError(f"unit parity exclusions contain duplicate nodeids: {sorted(duplicates)}") + return self + + +class RustUnitSpec(_ContractModel): + cargo_manifest: str + cargo_filter: str + cargo_package: str | None = None + + @field_validator("cargo_manifest", "cargo_filter") + @classmethod + def validate_required_fields(cls, value: str) -> str: + stripped: Final = value.strip() + if not stripped: + raise ValueError("must be a non-empty string") + return stripped + + @field_validator("cargo_package") + @classmethod + def validate_package(cls, value: str | None) -> str | None: + if value is None: + return None + stripped: Final = value.strip() + if not stripped: + raise ValueError("must be a non-empty string when provided") + return stripped + + +class UnitTestContract(_ContractModel): + unit_parity: UnitParitySpec + rust: RustUnitSpec + + +OCR_CONTRACT: Final = UnitTestContract( + unit_parity=UnitParitySpec( + python_selectors=( + "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", + "tests/test_litellm/llms/mistral/ocr", + "tests/test_litellm/llms/ocr", + "tests/test_litellm/ocr", + ), + exclusions=( + UnitParityExclusionSpec( + nodeid="tests/test_litellm/ocr/test_rust_bridge.py::test_rust_toggles_flag", + reason="This test asserts the process-level backend flag selected by the parity runner.", + ), + ), + ), + rust=RustUnitSpec( + cargo_manifest="litellm-rust/Cargo.toml", + cargo_filter="ocr", + ), +) + +UNIT_TEST_CONTRACTS: Final[Mapping[SdkFunction, UnitTestContract]] = MappingProxyType({"ocr": OCR_CONTRACT}) diff --git a/tests/rust-python-harness/strategies/trace_parity/AGENTS.md b/tests/rust-python-harness/strategies/trace_parity/AGENTS.md index 19861aea2b4..030caa557c8 100644 --- a/tests/rust-python-harness/strategies/trace_parity/AGENTS.md +++ b/tests/rust-python-harness/strategies/trace_parity/AGENTS.md @@ -1 +1 @@ -Prints every collected Python call under litellm/ and every feature-gated Rust span from live traces against replayed HTTP responses. The two traces are independent and are not compared. API-key and Vertex credentials scenarios exercise separate authentication paths; credentials scenarios replay the token exchange locally. +Prints every collected Python call under litellm/ from live traces against replayed HTTP responses. API-key and Vertex credentials scenarios exercise separate authentication paths; credentials scenarios replay the token exchange locally. diff --git a/tests/rust-python-harness/strategies/trace_parity/__init__.py b/tests/rust-python-harness/strategies/trace_parity/__init__.py index 710bdaa3d39..9aa4f46e4df 100644 --- a/tests/rust-python-harness/strategies/trace_parity/__init__.py +++ b/tests/rust-python-harness/strategies/trace_parity/__init__.py @@ -7,7 +7,6 @@ from ...shared.reporting.strategy import ( ModuleCaseSpec, NotImplementedCaseSpec, RunnerArgumentDefinition, - RunnerOptionDefinition, StrategyDefinition, ) from .reporting import render_trace_results @@ -72,20 +71,12 @@ CASES: Final[tuple[CaseDefinition, ...]] = ( ), CaseDefinition( "messages", - ModuleCaseSpec( - coverage=Coverage.PARTIAL, - module="tests.rust-python-harness.strategies.trace_parity.gateway.messages.case", - note="Anthropic/Azure provider routes plus a fully consumed downstream streaming path.", - ), + NotImplementedCaseSpec(reason="No gateway Messages trace-parity case is registered."), surface="gateway", ), CaseDefinition( "responses", - ModuleCaseSpec( - coverage=Coverage.PARTIAL, - module="tests.rust-python-harness.strategies.trace_parity.gateway.responses.case", - note="Native OpenAI non-streaming and fully consumed downstream streaming paths.", - ), + NotImplementedCaseSpec(reason="No gateway Responses trace-parity case is registered."), surface="gateway", ), CaseDefinition( @@ -95,11 +86,7 @@ CASES: Final[tuple[CaseDefinition, ...]] = ( ), CaseDefinition( "chat_completions", - ModuleCaseSpec( - coverage=Coverage.PARTIAL, - module="tests.rust-python-harness.strategies.trace_parity.gateway.chat_completions.case", - note="Anthropic non-streaming and fully consumed downstream streaming paths.", - ), + NotImplementedCaseSpec(reason="No gateway chat trace-parity case is registered."), surface="gateway", ), CaseDefinition( @@ -113,7 +100,7 @@ STRATEGY: Final = StrategyDefinition( id="trace_parity", order=20, label="Traces", - description="Print Python profiler frames and Rust spans for representative pipeline scenarios.", + description="Print Python profiler frames for representative pipeline scenarios.", directory=Path(__file__).parent, runnable_spec=ModuleCaseSpec, cases=CASES, @@ -125,11 +112,4 @@ STRATEGY: Final = StrategyDefinition( metavar="NAME", help="run only this named trace scenario; repeat to select more than one", ), - runner_options=( - RunnerOptionDefinition( - option="--engine", - choices=("python", "rust"), - help="show only this engine's trace; omit to print both engines", - ), - ), ) diff --git a/tests/rust-python-harness/strategies/trace_parity/gateway/__init__.py b/tests/rust-python-harness/strategies/trace_parity/gateway/__init__.py deleted file mode 100644 index f999dfecfc6..00000000000 --- a/tests/rust-python-harness/strategies/trace_parity/gateway/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""In-process gateway trace adapters.""" diff --git a/tests/rust-python-harness/strategies/trace_parity/gateway/chat_completions/case.py b/tests/rust-python-harness/strategies/trace_parity/gateway/chat_completions/case.py deleted file mode 100644 index 3dc6d731b4b..00000000000 --- a/tests/rust-python-harness/strategies/trace_parity/gateway/chat_completions/case.py +++ /dev/null @@ -1,63 +0,0 @@ -from __future__ import annotations - -from typing import Final - -from .....shared.tracing.steps import Engine, mapping -from ...fixtures import anthropic_response_body, anthropic_stream_events, json_response, sse_response -from ...models import GatewayRouteSpec, RouteFixture, TraceScenario, TraceSuite - -MAPPINGS: Final = ( - mapping(span="python_chat_gateway_route", python_frame=r"proxy_server\.py:\d+ chat_completion$"), - mapping(span="python_gateway_service", python_frame=r"ProxyBaseLLMRequestProcessing\.base_process_llm_request$"), - mapping(span="python_chat_entrypoint", python_frame=r"main\.py:\d+ a?completion$"), - mapping(span="python_provider_config", python_frame=r"ProviderConfigManager\.get_provider_chat_config$"), - mapping(rust_span="validate_environment", python_frame=r"(? RouteFixture: - return RouteFixture( - kwargs={ - "model_alias": "trace-model", - "provider_model": "anthropic/claude-sonnet-5", - "body": { - "model": "trace-model", - "messages": [{"role": "user", "content": "hello"}], - "max_tokens": 16, - }, - }, - provider_responses=(json_response(anthropic_response_body()),), - ) - - -def _stream_fixture(engine: Engine, base_url: str) -> RouteFixture: - fixture: Final = _fixture(engine, base_url) - return fixture.with_body(stream=True).derive( - provider_responses=(sse_response(anthropic_stream_events()),), - ) - - -TRACE_SUITE: Final = TraceSuite( - route=GatewayRouteSpec("chat_completions", rust_supported=False), - scenarios=( - TraceScenario(name="async-anthropic", fixture=_fixture, mappings=MAPPINGS, asynchronous=True), - TraceScenario( - name="async-anthropic-downstream-stream", - fixture=_stream_fixture, - mappings=(*MAPPINGS, *STREAM_MAPPINGS), - asynchronous=True, - ), - ), -) diff --git a/tests/rust-python-harness/strategies/trace_parity/gateway/execution.py b/tests/rust-python-harness/strategies/trace_parity/gateway/execution.py deleted file mode 100644 index 94d6be7cebf..00000000000 --- a/tests/rust-python-harness/strategies/trace_parity/gateway/execution.py +++ /dev/null @@ -1,197 +0,0 @@ -from __future__ import annotations - -import json -import subprocess -from functools import cache -from pathlib import Path -from typing import Final, Protocol, cast - -import httpx -from pydantic import BaseModel, ConfigDict - -from ....shared.parity.replay import replay_server -from ....shared.tracing.native import TraceResponsePayload, native_trace_events -from ....shared.tracing.profiler import FunctionTraceEvent, profile_python -from ....shared.tracing.steps import Engine, PipelineProjection, pipeline_projection -from ..models import GatewayRouteSpec, RouteFixture, TraceEngine, TraceExecutionFailure, TraceScenario -from ..reporting import TraceArtifact - - -class _GatewayResponsePayload(BaseModel): - model_config = ConfigDict(strict=True, extra="forbid") - - status: int - body: object - - -class _GatewayClient(Protocol): - def post(self, url: str, *, json: object, headers: dict[str, str]) -> httpx.Response: ... - - -_ROUTE_PATHS: Final = { - "messages": "/v1/messages", - "chat_completions": "/v1/chat/completions", - "responses": "/v1/responses", -} - - -def _collect_python(fixture: RouteFixture, route: GatewayRouteSpec) -> tuple[FunctionTraceEvent, ...]: - from fastapi.testclient import TestClient - - import litellm - from litellm.proxy import proxy_server - from litellm.proxy._types import UserAPIKeyAuth - from litellm.proxy.anthropic_endpoints.endpoints import user_api_key_auth - - provider_model: Final = cast(str, fixture.kwargs["provider_model"]) - model_alias: Final = cast(str, fixture.kwargs["model_alias"]) - old_router: Final = proxy_server.llm_router - old_override: Final = proxy_server.app.dependency_overrides.get(user_api_key_auth) - - async def authorize() -> UserAPIKeyAuth: - return UserAPIKeyAuth(api_key="trace-key") - - proxy_server.llm_router = litellm.Router( - model_list=[ - { - "model_name": model_alias, - "litellm_params": { - "model": provider_model, - "api_key": "trace-provider-key", - "api_base": fixture.kwargs["api_base"], - }, - } - ] - ) - proxy_server.app.dependency_overrides[user_api_key_auth] = authorize - try: - with profile_python(Path(litellm.__file__).parent, threads=True) as profiler: - client: Final = cast(_GatewayClient, TestClient(proxy_server.app)) - response: Final = client.post( - _ROUTE_PATHS[route.route], - json=fixture.kwargs["body"], - headers={"authorization": "Bearer trace-key"}, - ) - if response.status_code != 200: - raise RuntimeError(f"Python gateway returned {response.status_code}: {response.text}") - return tuple(profiler.events) - finally: - proxy_server.llm_router = old_router - if old_override is None: - proxy_server.app.dependency_overrides.pop(user_api_key_auth, None) - else: - proxy_server.app.dependency_overrides[user_api_key_auth] = old_override - - -def _collect_rust(fixture: RouteFixture, route: GatewayRouteSpec) -> tuple[FunctionTraceEvent, ...]: - payload: Final = json.dumps( - { - "path": _ROUTE_PATHS[route.route], - "model_alias": fixture.kwargs["model_alias"], - "provider_model": fixture.kwargs["provider_model"], - "api_base": fixture.kwargs["api_base"], - "body": fixture.kwargs["body"], - } - ) - completed: Final = subprocess.run( - (_gateway_trace_binary(),), - input=payload, - capture_output=True, - text=True, - check=False, - ) - if completed.returncode != 0: - raise RuntimeError(f"Rust gateway trace failed: {completed.stderr.strip()}") - result: Final = json.loads(completed.stdout) - payload: Final = TraceResponsePayload.model_validate(result) - response: Final = _GatewayResponsePayload.model_validate(payload.response) - if response.status != 200: - raise RuntimeError(f"Rust gateway returned {response.status}: {response.body}") - return native_trace_events(payload) - - -@cache -def _gateway_trace_binary() -> Path: - repo_root: Final = next(parent for parent in Path(__file__).resolve().parents if (parent / "litellm-rust").is_dir()) - rust_root: Final = repo_root / "litellm-rust" - completed: Final = subprocess.run( - ( - "cargo", - "build", - "--quiet", - "--package", - "litellm-ai-gateway", - "--features", - "trace-parity", - "--bin", - "trace-parity-gateway", - "--target-dir", - rust_root / "target", - ), - cwd=rust_root, - capture_output=True, - text=True, - check=False, - ) - if completed.returncode != 0: - raise RuntimeError(f"Rust gateway trace build failed: {completed.stderr.strip()}") - return rust_root / "target" / "debug" / "trace-parity-gateway" - - -def _collect( - route: GatewayRouteSpec, scenario: TraceScenario, engine: Engine -) -> tuple[FunctionTraceEvent, ...] | TraceExecutionFailure: - try: - with replay_server() as provider: - base_fixture: Final = scenario.fixture(engine, provider.url) - fixture: Final = RouteFixture( - kwargs={**base_fixture.kwargs, "api_base": provider.url}, - provider_responses=base_fixture.provider_responses, - ) - for response in fixture.provider_responses: - provider.enqueue_response(response) - events: Final = _collect_python(fixture, route) if engine == "python" else _collect_rust(fixture, route) - provider.take_requests(len(fixture.provider_responses)) - return events - except Exception as error: - return TraceExecutionFailure(engine, f"{type(error).__name__}: {error}") - - -def _projections( - python_events: tuple[FunctionTraceEvent, ...], - rust_events: tuple[FunctionTraceEvent, ...], -) -> tuple[PipelineProjection, PipelineProjection, str | None]: - try: - return ( - pipeline_projection("python", python_events), - pipeline_projection("rust", rust_events), - None, - ) - except ValueError as error: - return PipelineProjection(), PipelineProjection(), f"harness: {error}" - - -def execute_gateway_trace( - route: GatewayRouteSpec, - scenario: TraceScenario, - engine: TraceEngine = "both", -) -> TraceArtifact: - effective_engine: Final[TraceEngine] = "python" if engine == "both" and not route.rust_supported else engine - python_trace: Final = _collect(route, scenario, "python") if effective_engine != "rust" else () - rust_trace: Final = _collect(route, scenario, "rust") if effective_engine != "python" else () - collection_python_error: Final = None if isinstance(python_trace, tuple) else f"python: {python_trace.message}" - rust_error: Final = None if isinstance(rust_trace, tuple) else f"rust: {rust_trace.message}" - python_events: Final = python_trace if isinstance(python_trace, tuple) else () - rust_events: Final = rust_trace if isinstance(rust_trace, tuple) else () - python, rust, projection_error = _projections(python_events, rust_events) - python_error: Final = projection_error or collection_python_error - return TraceArtifact.from_traces( - engine=effective_engine, - surface="gateway", - sdk_function=route.route, - scenario=scenario.name, - python=python.steps, - rust=rust.steps, - python_error=python_error, - rust_error=rust_error, - ) diff --git a/tests/rust-python-harness/strategies/trace_parity/gateway/messages/__init__.py b/tests/rust-python-harness/strategies/trace_parity/gateway/messages/__init__.py deleted file mode 100644 index bd9195b7c22..00000000000 --- a/tests/rust-python-harness/strategies/trace_parity/gateway/messages/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Messages gateway trace cases.""" diff --git a/tests/rust-python-harness/strategies/trace_parity/gateway/messages/case.py b/tests/rust-python-harness/strategies/trace_parity/gateway/messages/case.py deleted file mode 100644 index ca9c858f6b7..00000000000 --- a/tests/rust-python-harness/strategies/trace_parity/gateway/messages/case.py +++ /dev/null @@ -1,110 +0,0 @@ -from __future__ import annotations - -from typing import Final - -from .....shared.tracing.steps import Engine, mapping -from ...fixtures import anthropic_response_body, anthropic_stream_events, json_response, sse_response -from ...models import GatewayRouteSpec, RouteFixture, TraceScenario, TraceSuite - - -GATEWAY_MAPPINGS: Final = ( - mapping( - span="python_messages_gateway_route", - python_frame=r"anthropic_endpoints/endpoints\.py:\d+ anthropic_response$", - ), - mapping(rust_span="messages_gateway_route"), - mapping( - span="python_messages_gateway_service", - python_frame=r"ProxyBaseLLMRequestProcessing\.base_process_llm_request$", - ), - mapping(rust_span="messages_gateway_service"), - mapping(rust_span="messages"), - mapping( - span="python_messages_provider_config", - python_frame=r"ProviderConfigManager\.get_provider_anthropic_messages_config$", - ), - mapping(rust_span="messages_provider_config"), - mapping(rust_span="validate_environment", python_frame=r"validate_anthropic_messages_environment$"), - mapping(rust_span="complete_url", python_frame=r"get_complete_url$"), - mapping(span="python_messages_entry_handler", python_frame=r"messages/handler\.py:\d+ anthropic_messages_handler$"), - mapping(span="python_messages_handler_wrapper", python_frame=r"BaseLLMHTTPHandler\.anthropic_messages_handler$"), - mapping( - rust_span="execute_messages_provider_call", - python_frame=r"BaseLLMHTTPHandler\.async_anthropic_messages_handler$", - ), - mapping(rust_span="http_request", python_frame=r"AsyncHTTPHandler\.post$|HTTPHandler\.post$"), - mapping(rust_span="transform_response", python_frame=r"(? RouteFixture: - return RouteFixture( - kwargs={ - "model_alias": "trace-model", - "provider_model": f"{provider}/claude-sonnet-5", - "body": { - "model": "trace-model", - "messages": [{"role": "user", "content": "hello"}], - "max_tokens": 16, - }, - }, - provider_responses=(json_response(anthropic_response_body()),), - ) - - -def _anthropic_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _fixture(engine, "anthropic") - - -def _azure_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _fixture(engine, "azure_ai") - - -def _stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _anthropic_fixture(engine, _base_url) - return fixture.with_body(stream=True).derive( - provider_responses=(sse_response(anthropic_stream_events()),), - ) - - -ANTHROPIC_MAPPINGS: Final = ( - *GATEWAY_MAPPINGS, - mapping( - rust_span="transform_request", - python_frame=r"(? RouteFixture: - return RouteFixture( - kwargs={ - "model_alias": "trace-model", - "provider_model": "openai/gpt-5", - "body": {"model": "trace-model", "input": "hello"}, - }, - provider_responses=(json_response(responses_body()),), - ) - - -def _stream_fixture(engine: Engine, base_url: str) -> RouteFixture: - fixture: Final = _fixture(engine, base_url) - return fixture.with_body(stream=True).derive( - provider_responses=(sse_response(responses_stream_events()),), - ) - - -TRACE_SUITE: Final = TraceSuite( - route=GatewayRouteSpec("responses", rust_supported=False), - scenarios=( - TraceScenario(name="async-openai", fixture=_fixture, mappings=MAPPINGS, asynchronous=True), - TraceScenario( - name="async-openai-downstream-stream", - fixture=_stream_fixture, - mappings=(*MAPPINGS, *STREAM_MAPPINGS), - asynchronous=True, - ), - ), -) diff --git a/tests/rust-python-harness/strategies/trace_parity/models.py b/tests/rust-python-harness/strategies/trace_parity/models.py index d6ed42250c4..7e6fc321d93 100644 --- a/tests/rust-python-harness/strategies/trace_parity/models.py +++ b/tests/rust-python-harness/strategies/trace_parity/models.py @@ -2,14 +2,12 @@ from __future__ import annotations from collections.abc import Callable, Mapping from dataclasses import dataclass -from typing import Final, Literal, TypeAlias, cast +from typing import Final, Literal, cast from ...shared.parity.recorded_http import RecordedResponse from ...shared.reporting.models import SdkFunction -from ...shared.tracing.steps import Engine, TraceMapping -TraceEngine = Literal["python", "rust", "both"] -TraceFailureSource = Literal["python", "rust", "harness"] +TraceFailureSource = Literal["python", "harness"] @dataclass(frozen=True, slots=True) @@ -48,30 +46,19 @@ class RouteFixture: class RouteSpec: route: SdkFunction python_entrypoints: tuple[str, str] - rust_entrypoints: tuple[str, str] | None - fixture: Callable[[Engine, str], RouteFixture] - - -@dataclass(frozen=True, slots=True) -class GatewayRouteSpec: - route: SdkFunction - rust_supported: bool = True - - -TraceRouteSpec: TypeAlias = RouteSpec | GatewayRouteSpec + fixture: Callable[[str], RouteFixture] @dataclass(frozen=True, slots=True) class TraceScenario: name: str - fixture: Callable[[Engine, str], RouteFixture] - mappings: tuple[TraceMapping, ...] + fixture: Callable[[str], RouteFixture] asynchronous: bool @dataclass(frozen=True, slots=True) class TraceSuite: - route: TraceRouteSpec + route: RouteSpec scenarios: tuple[TraceScenario, ...] diff --git a/tests/rust-python-harness/strategies/trace_parity/reporting.py b/tests/rust-python-harness/strategies/trace_parity/reporting.py index e7c07ef9c0f..086cf2d03c7 100644 --- a/tests/rust-python-harness/strategies/trace_parity/reporting.py +++ b/tests/rust-python-harness/strategies/trace_parity/reporting.py @@ -11,14 +11,10 @@ from ...shared.reporting.models import SURFACES, CaseResult, RunStatus, SdkFunct from ...shared.reporting.rendering import ReportSection from ...shared.reporting.strategy import NotImplementedCaseSpec, SkippedCaseSpec from ...shared.tracing.steps import PipelineStep, trace_depths -from .models import TraceEngine TRACE_ARTIFACT: Final = "trace" -TRACE_PARITY_HINT: Final = ( - "rebuild the native bridge with the trace-parity feature, e.g. `uvx maturin develop --features trace-parity`" -) -_COLORS: Final[dict[str, str]] = {"yellow": "33", "red": "31", "cyan": "36"} +_COLORS: Final[dict[str, str]] = {"red": "31", "cyan": "36"} _RESET: Final = "\033[0m" @@ -43,30 +39,23 @@ class TraceEventArtifact(BaseModel): class TraceArtifact(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") - engine: TraceEngine = "both" surface: Surface sdk_function: SdkFunction scenario: str python: tuple[TraceEventArtifact, ...] - rust: tuple[TraceEventArtifact, ...] python_error: str | None = None - rust_error: str | None = None @classmethod def from_traces( cls, *, - engine: TraceEngine = "both", surface: Surface, sdk_function: SdkFunction, scenario: str, python: Sequence[PipelineStep], - rust: Sequence[PipelineStep], python_error: str | None = None, - rust_error: str | None = None, ) -> TraceArtifact: return cls( - engine=engine, surface=surface, sdk_function=sdk_function, scenario=scenario, @@ -74,21 +63,14 @@ class TraceArtifact(BaseModel): TraceEventArtifact(id=step.id, parent_id=step.parent_id, span=step.span, raw=step.raw) for step in python ), - rust=tuple( - TraceEventArtifact(id=step.id, parent_id=step.parent_id, span=step.span, raw=step.raw) for step in rust - ), python_error=python_error, - rust_error=rust_error, ) def python_steps(self) -> tuple[PipelineStep, ...]: return tuple(event.step() for event in self.python) - def rust_steps(self) -> tuple[PipelineStep, ...]: - return tuple(event.step() for event in self.rust) - def has_errors(self) -> bool: - return self.python_error is not None or self.rust_error is not None + return self.python_error is not None def _split_raw(raw: str) -> tuple[str, str]: @@ -111,34 +93,14 @@ def _python_lines(steps: tuple[PipelineStep, ...]) -> str: return f"{_paint('PYTHON', 'cyan')} ({len(steps)} steps)\n" + ("\n".join(lines) if lines else "(empty)") -def _rust_lines(steps: tuple[PipelineStep, ...]) -> str: - depths: Final = trace_depths(steps) - lines: Final = tuple( - _paint(f"{index} {' ' * depths[step.id]}{step.span}", "yellow") for index, step in enumerate(steps, 1) - ) - return f"{_paint('RUST', 'yellow')} ({len(steps)} steps)\n" + ("\n".join(lines) if lines else "(empty)") - - def _error_lines(artifact: TraceArtifact) -> tuple[str, ...]: - lines: list[str] = [] - for engine, error in (("Python", artifact.python_error), ("Rust", artifact.rust_error)): - if error is None: - continue - lines.append(_paint(f"{engine} error: {error}", "red")) - if "trace-parity feature" in error: - lines.append(f"hint: {TRACE_PARITY_HINT}") - return tuple(lines) + if artifact.python_error is None: + return () + return (_paint(f"Python error: {artifact.python_error}", "red"),) def _render_trace(artifact: TraceArtifact) -> str: - traces: tuple[str, ...] - if artifact.engine == "python": - traces = (_python_lines(artifact.python_steps()),) - elif artifact.engine == "rust": - traces = (_rust_lines(artifact.rust_steps()),) - else: - traces = (_python_lines(artifact.python_steps()), _rust_lines(artifact.rust_steps())) - return "\n\n".join((*traces, *_error_lines(artifact))) + return "\n\n".join((_python_lines(artifact.python_steps()), *_error_lines(artifact))) def _scenario(nodeid: str) -> str: diff --git a/tests/rust-python-harness/strategies/trace_parity/runner.py b/tests/rust-python-harness/strategies/trace_parity/runner.py index 706e054bf52..9147bcfe8b3 100644 --- a/tests/rust-python-harness/strategies/trace_parity/runner.py +++ b/tests/rust-python-harness/strategies/trace_parity/runner.py @@ -4,15 +4,11 @@ import importlib from collections.abc import Sequence from pathlib import Path from time import monotonic -from typing import Final, cast +from typing import Final -from ...shared.native_build import ensure_trace_bridge from ...shared.reporting.models import CaseResult, HarnessCase, HarnessRun, ResultArtifact, RunStatus, Surface from ...shared.reporting.strategy import ModuleCaseSpec, UpdateCallback from .models import ( - GatewayRouteSpec, - RouteSpec, - TraceEngine, TraceExecutionFailure, TraceScenario, TraceSuite, @@ -47,12 +43,8 @@ def validate_trace_suite(suite: TraceSuite, harness_case: HarnessCase) -> str | if invalid_names: return f"scenario names must start with sync- or async-: {', '.join(invalid_names)}" surface: Final = harness_case.surface - if surface == "sdk" and not isinstance(suite.route, RouteSpec): - return "must use RouteSpec for the sdk surface" - if surface == "gateway" and not isinstance(suite.route, GatewayRouteSpec): - return "must use GatewayRouteSpec for the gateway surface" - if surface is None: - return "requires an sdk or gateway surface" + if surface != "sdk": + return "requires the sdk surface" if suite.route.route != harness_case.sdk_function: return f"route {suite.route.route} does not match case function {harness_case.sdk_function}" return None @@ -89,10 +81,9 @@ def run_trace_scenario( surface: Surface, nodeid: str, on_update: UpdateCallback, - engine: TraceEngine = "both", ) -> None: started_at: Final = monotonic() - trace: Final = _execute_scenario(trace_suite, scenario, surface, engine) + trace: Final = _execute_scenario(trace_suite, scenario, surface) duration: Final = monotonic() - started_at if isinstance(trace, TraceExecutionFailure): result.record(nodeid, RunStatus.ERROR, duration) @@ -102,7 +93,7 @@ def run_trace_scenario( artifact: Final = ResultArtifact(TRACE_ARTIFACT, trace.model_dump_json()) if trace.has_errors(): result.record(nodeid, RunStatus.ERROR, duration, (artifact,)) - run.failures.append((nodeid, "\n".join(error for error in (trace.python_error, trace.rust_error) if error))) + run.failures.append((nodeid, trace.python_error or "")) else: result.record(nodeid, RunStatus.PASSED, duration, (artifact,)) on_update(run) @@ -112,18 +103,10 @@ def _execute_scenario( trace_suite: TraceSuite, scenario: TraceScenario, surface: Surface, - engine: TraceEngine, ) -> TraceArtifact | TraceExecutionFailure: - route: Final = trace_suite.route - if isinstance(route, GatewayRouteSpec): - if surface != "gateway": - return TraceExecutionFailure("harness", "gateway route cannot run on the sdk surface") - from .gateway.execution import execute_gateway_trace - - return execute_gateway_trace(route, scenario, engine) if surface != "sdk": - return TraceExecutionFailure("harness", "sdk route cannot run on the gateway surface") - return execute_trace(route, scenario, surface, engine) + return TraceExecutionFailure("harness", "trace scenarios only run on the sdk surface") + return execute_trace(trace_suite.route, scenario, surface) def _run_case( @@ -131,7 +114,6 @@ def _run_case( harness_case: HarnessCase, selected_scenarios: frozenset[str], on_update: UpdateCallback, - engine: TraceEngine, ) -> None: result: Final = run.results[harness_case.key] spec: Final = harness_case.spec @@ -154,21 +136,7 @@ def _run_case( result.status = RunStatus.RUNNING on_update(run) for scenario, nodeid in nodeids: - run_trace_scenario(run, result, trace_suite, scenario, surface, nodeid, on_update, engine) - - -def runner_selection(runner_args: Sequence[str]) -> tuple[frozenset[str], TraceEngine]: - engine: TraceEngine = "both" - scenarios: list[str] = [] - for argument in runner_args: - if argument.startswith("--engine="): - value = argument.removeprefix("--engine=") - if value not in {"python", "rust"}: - raise ValueError(f"invalid trace engine: {value}") - engine = cast(TraceEngine, value) - else: - scenarios.append(argument) - return frozenset(scenarios), engine + run_trace_scenario(run, result, trace_suite, scenario, surface, nodeid, on_update) def run_trace_cases( @@ -177,18 +145,11 @@ def run_trace_cases( on_update: UpdateCallback, runner_args: Sequence[str] = (), ) -> tuple[int, HarnessRun]: - selected_scenarios, engine = runner_selection(runner_args) + del repo_root + selected_scenarios: Final = frozenset(runner_args) run: Final = HarnessRun.from_cases(cases) - runnable_cases: Final = tuple(case for case in cases if isinstance(case.spec, ModuleCaseSpec)) - bridge_error: Final = ensure_trace_bridge(repo_root) if runnable_cases and engine != "python" else None - if bridge_error is not None: - for harness_case in runnable_cases: - _record_setup_failure(run, harness_case, bridge_error, "bridge") - run.finished_at = monotonic() - on_update(run) - return 1, run for harness_case in cases: - _run_case(run, harness_case, selected_scenarios, on_update, engine) + _run_case(run, harness_case, selected_scenarios, on_update) run.finished_at = monotonic() on_update(run) failed: Final = any( diff --git a/tests/rust-python-harness/strategies/trace_parity/sdk/chat_completions/case.py b/tests/rust-python-harness/strategies/trace_parity/sdk/chat_completions/case.py index 1221f237570..016a3683079 100644 --- a/tests/rust-python-harness/strategies/trace_parity/sdk/chat_completions/case.py +++ b/tests/rust-python-harness/strategies/trace_parity/sdk/chat_completions/case.py @@ -2,7 +2,6 @@ from __future__ import annotations from typing import Final -from .....shared.tracing.steps import Engine, mapping from ...fixtures import ( anthropic_response_body, anthropic_stream_events, @@ -12,70 +11,19 @@ from ...fixtures import ( ) from ...models import RouteFixture, RouteSpec, TraceScenario, TraceSuite -COMMON_MAPPINGS: Final = ( - mapping(span="python_provider_config", python_frame=r"ProviderConfigManager\.get_provider_chat_config$"), - mapping(rust_span="chat_completions_provider_config"), - mapping( - span="python_supported_openai_params", - python_frame=r"litellm_core_utils/get_supported_openai_params\.py:\d+ get_supported_openai_params$", - ), - mapping( - span="python_provider_supported_openai_params", - python_frame=r"AnthropicConfig\.get_supported_openai_params$", - ), - mapping(rust_span="supported_openai_params"), - mapping(rust_span="validate_environment", python_frame=r"(? RouteFixture: +def _anthropic_fixture(_base_url: str) -> RouteFixture: return RouteFixture( kwargs={ "model": "anthropic/claude-sonnet-5", "messages": [{"role": "user", "content": "hello"}], - **({"optional_params": {"max_tokens": 16}} if engine == "rust" else {"max_tokens": 16}), + "max_tokens": 16, }, provider_responses=(json_response(anthropic_response_body()),), ) -def _bedrock_fixture(engine: Engine, _base_url: str) -> RouteFixture: +def _bedrock_fixture(_base_url: str) -> RouteFixture: response: Final[dict[str, object]] = { "output": {"message": {"role": "assistant", "content": [{"text": "hello"}]}}, "stopReason": "end_turn", @@ -91,18 +39,15 @@ def _bedrock_fixture(engine: Engine, _base_url: str) -> RouteFixture: kwargs={ "model": "bedrock/us-east-1/anthropic.claude-v2", "messages": [{"role": "user", "content": "hello"}], - **( - {"optional_params": {**credentials, "maxTokens": 16}} - if engine == "rust" - else {**credentials, "max_tokens": 16} - ), + **credentials, + "max_tokens": 16, }, provider_responses=(json_response(response),), ) -def _anthropic_stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _anthropic_fixture(engine, _base_url) +def _anthropic_stream_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _anthropic_fixture(_base_url) return fixture.derive( kwargs={"stream": True}, provider_responses=(sse_response(anthropic_stream_events()),), @@ -110,8 +55,8 @@ def _anthropic_stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _bedrock_stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _bedrock_fixture(engine, _base_url) +def _bedrock_stream_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _bedrock_fixture(_base_url) events: Final[tuple[dict[str, object], ...]] = ( {"messageStart": {"role": "assistant"}}, {"contentBlockStart": {"contentBlockIndex": 0, "start": {}}}, @@ -127,8 +72,8 @@ def _bedrock_stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _provider_error_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _anthropic_fixture(engine, _base_url) +def _provider_error_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _anthropic_fixture(_base_url) return fixture.derive( provider_responses=( json_response( @@ -140,8 +85,8 @@ def _provider_error_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _stream_error_fixture(engine: Engine, base_url: str) -> RouteFixture: - fixture: Final = _anthropic_fixture(engine, base_url) +def _stream_error_fixture(base_url: str) -> RouteFixture: + fixture: Final = _anthropic_fixture(base_url) events: Final = ( anthropic_stream_events()[0], ("error", {"type": "error", "error": {"type": "overloaded_error", "message": "overloaded"}}), @@ -157,90 +102,59 @@ def _stream_error_fixture(engine: Engine, base_url: str) -> RouteFixture: SPEC: Final = RouteSpec( "chat_completions", ("completion", "acompletion"), - ("chat_completions", "achat_completions"), _anthropic_fixture, ) -BEDROCK_COMMON_MAPPINGS: Final = ( - mapping(rust_span="chat_completions_provider_config"), - mapping(rust_span="supported_openai_params"), - mapping(rust_span="execute_chat_completions_provider_call"), - mapping(rust_span="validate_environment"), - mapping(rust_span="http_request", python_frame=r"AsyncHTTPHandler\.post$|HTTPHandler\.post$"), - mapping(span="python_transform_response", python_frame=r"AmazonConverseConfig\._transform_response$"), -) -BEDROCK_SYNC_MAPPINGS: Final = ( - mapping(span="python_chat_completions", python_frame=r"main\.py:\d+ completion$"), - mapping(rust_span="chat_completions"), - mapping(span="python_transform_request", python_frame=r"AmazonConverseConfig\._transform_request$"), - *BEDROCK_COMMON_MAPPINGS, -) -BEDROCK_ASYNC_MAPPINGS: Final = ( - mapping(span="python_chat_completions", python_frame=r"main\.py:\d+ acompletion$"), - mapping(span="python_completion_wrapper", python_frame=r"main\.py:\d+ completion$"), - mapping(rust_span="chat_completions"), - *BEDROCK_COMMON_MAPPINGS, -) TRACE_SUITE: Final = TraceSuite( route=SPEC, scenarios=( TraceScenario( name="sync-anthropic", fixture=_anthropic_fixture, - mappings=SYNC_MAPPINGS, asynchronous=False, ), TraceScenario( name="async-anthropic", fixture=_anthropic_fixture, - mappings=ASYNC_MAPPINGS, asynchronous=True, ), TraceScenario( name="sync-anthropic-stream", fixture=_anthropic_stream_fixture, - mappings=(*SYNC_MAPPINGS, *STREAM_MAPPINGS), asynchronous=False, ), TraceScenario( name="async-anthropic-stream", fixture=_anthropic_stream_fixture, - mappings=(*ASYNC_MAPPINGS, *STREAM_MAPPINGS), asynchronous=True, ), TraceScenario( name="async-anthropic-provider-error", fixture=_provider_error_fixture, - mappings=(*ASYNC_MAPPINGS, *FAILURE_MAPPINGS), asynchronous=True, ), TraceScenario( name="async-anthropic-stream-error", fixture=_stream_error_fixture, - mappings=(*ASYNC_MAPPINGS, *STREAM_MAPPINGS, *FAILURE_MAPPINGS), asynchronous=True, ), TraceScenario( name="sync-bedrock", fixture=_bedrock_fixture, - mappings=BEDROCK_SYNC_MAPPINGS, asynchronous=False, ), TraceScenario( name="async-bedrock", fixture=_bedrock_fixture, - mappings=BEDROCK_ASYNC_MAPPINGS, asynchronous=True, ), TraceScenario( name="sync-bedrock-event-stream", fixture=_bedrock_stream_fixture, - mappings=(*BEDROCK_SYNC_MAPPINGS, *STREAM_MAPPINGS), asynchronous=False, ), TraceScenario( name="async-bedrock-event-stream", fixture=_bedrock_stream_fixture, - mappings=(*BEDROCK_ASYNC_MAPPINGS, *STREAM_MAPPINGS), asynchronous=True, ), ), diff --git a/tests/rust-python-harness/strategies/trace_parity/sdk/execution.py b/tests/rust-python-harness/strategies/trace_parity/sdk/execution.py index 783c22a0dc0..ed98e550f4e 100644 --- a/tests/rust-python-harness/strategies/trace_parity/sdk/execution.py +++ b/tests/rust-python-harness/strategies/trace_parity/sdk/execution.py @@ -10,10 +10,9 @@ from unittest.mock import patch from ....shared.parity.replay import replay_server from ....shared.reporting.models import Surface -from ....shared.tracing.native import TraceResponsePayload, native_trace_events from ....shared.tracing.profiler import FunctionTraceEvent, profile_python -from ....shared.tracing.steps import Engine, pipeline_projection -from ..models import RouteFixture, RouteSpec, TraceEngine, TraceExecutionFailure, TraceScenario +from ....shared.tracing.steps import pipeline_projection +from ..models import RouteFixture, RouteSpec, TraceExecutionFailure, TraceScenario from ..reporting import TraceArtifact @@ -56,25 +55,10 @@ def _invoke( return response -def _entrypoint(spec: RouteSpec, engine: Engine, *, asynchronous: bool) -> SdkCall | TraceExecutionFailure: +def _entrypoint(spec: RouteSpec, *, asynchronous: bool) -> SdkCall: import litellm from litellm.anthropic_interface import messages as sdk_messages - from litellm.rust_bridge import get_native_bridge - if engine == "rust": - if spec.rust_entrypoints is None: - return TraceExecutionFailure("rust", f"{spec.route} has no native Rust trace entrypoint") - bridge: Final = cast(object | None, get_native_bridge()) - if bridge is None: - return TraceExecutionFailure("rust", "native Rust bridge is required for trace parity") - trace_bridge: Final[object | None] = getattr(bridge, "_trace", None) - if trace_bridge is None: - return TraceExecutionFailure("rust", "native Rust bridge must include the trace-parity feature") - entrypoint: Final = spec.rust_entrypoints[int(asynchronous)] - function: Final[object | None] = getattr(trace_bridge, entrypoint, None) - if function is None: - return TraceExecutionFailure("rust", f"native Rust trace bridge does not expose {entrypoint}") - return cast(SdkCall, function) owner: Final = sdk_messages if spec.route == "messages" else litellm return cast(SdkCall, getattr(owner, spec.python_entrypoints[int(asynchronous)])) @@ -82,14 +66,9 @@ def _entrypoint(spec: RouteSpec, engine: Engine, *, asynchronous: bool) -> SdkCa def _collect( function: SdkCall, fixture: RouteFixture, - engine: Engine, *, asynchronous: bool, ) -> _CollectedTrace: - kwargs: Final = fixture.kwargs - if engine == "rust": - payload: Final = TraceResponsePayload.model_validate(_invoke(function, kwargs, asynchronous=asynchronous)) - return _CollectedTrace(native_trace_events(payload), payload.error) import litellm previous_suppress_debug_info: Final = litellm.suppress_debug_info @@ -99,7 +78,7 @@ def _collect( with profile_python(Path(litellm.__file__).parent, threads=True) as profiler: error: str | None try: - _invoke(function, kwargs, asynchronous=asynchronous, consume_stream=fixture.consume_stream) + _invoke(function, fixture.kwargs, asynchronous=asynchronous, consume_stream=fixture.consume_stream) error = None except Exception as caught: error = f"{type(caught).__name__}: {caught}" @@ -108,13 +87,11 @@ def _collect( return _CollectedTrace(tuple(profiler.events), error) -def collect_trace(spec: RouteSpec, engine: Engine, *, asynchronous: bool) -> tuple[FunctionTraceEvent, ...] | TraceExecutionFailure: - function: Final = _entrypoint(spec, engine, asynchronous=asynchronous) - if isinstance(function, TraceExecutionFailure): - return function +def collect_trace(spec: RouteSpec, *, asynchronous: bool) -> tuple[FunctionTraceEvent, ...] | TraceExecutionFailure: + function: Final = _entrypoint(spec, asynchronous=asynchronous) try: with replay_server() as provider: - base_fixture: Final = spec.fixture(engine, provider.url) + base_fixture: Final = spec.fixture(provider.url) for response in base_fixture.provider_responses: provider.enqueue_response(response) fixture: Final = RouteFixture( @@ -122,7 +99,7 @@ def collect_trace(spec: RouteSpec, engine: Engine, *, asynchronous: bool) -> tup "api_key": "test-key", **base_fixture.kwargs, "api_base": provider.url, - **({"timeout_seconds": 5} if engine == "rust" else {"timeout": 5}), + "timeout": 5, }, provider_responses=base_fixture.provider_responses, expected_failure=base_fixture.expected_failure, @@ -130,76 +107,42 @@ def collect_trace(spec: RouteSpec, engine: Engine, *, asynchronous: bool) -> tup environment=base_fixture.environment, ) with patch.dict(os.environ, fixture.environment): - collected: Final = _collect(function, fixture, engine, asynchronous=asynchronous) + collected: Final = _collect(function, fixture, asynchronous=asynchronous) provider.take_requests(len(fixture.provider_responses)) except Exception as error: - return TraceExecutionFailure(engine, f"{type(error).__name__}: {error}") + return TraceExecutionFailure("python", f"{type(error).__name__}: {error}") if fixture.expected_failure and collected.error is None: - return TraceExecutionFailure(engine, "call succeeded but the scenario expects failure") + return TraceExecutionFailure("python", "call succeeded but the scenario expects failure") if not fixture.expected_failure and collected.error is not None: - return TraceExecutionFailure(engine, collected.error) + return TraceExecutionFailure("python", collected.error) if not collected.events: - return TraceExecutionFailure(engine, "trace is empty") + return TraceExecutionFailure("python", "trace is empty") return collected.events -def _failure_message(result: tuple[FunctionTraceEvent, ...] | TraceExecutionFailure) -> str | None: - if isinstance(result, tuple): - return None - return f"{result.engine}: {result.message}" - - -def execute_trace( - route: RouteSpec, - scenario: TraceScenario, - surface: Surface, - engine: TraceEngine = "both", -) -> TraceArtifact: - effective_engine: Final[TraceEngine] = "python" if engine == "both" and route.rust_entrypoints is None else engine +def execute_trace(route: RouteSpec, scenario: TraceScenario, surface: Surface) -> TraceArtifact: scenario_route: Final = RouteSpec( route=route.route, python_entrypoints=route.python_entrypoints, - rust_entrypoints=route.rust_entrypoints, fixture=scenario.fixture, ) - python_trace: Final = ( - collect_trace( - scenario_route, - "python", - asynchronous=scenario.asynchronous, - ) - if effective_engine != "rust" - else () - ) - rust_trace: Final = ( - collect_trace(scenario_route, "rust", asynchronous=scenario.asynchronous) - if effective_engine != "python" - else () - ) - python_error: Final = _failure_message(python_trace) - rust_error: Final = _failure_message(rust_trace) + python_trace: Final = collect_trace(scenario_route, asynchronous=scenario.asynchronous) + python_error: Final = None if isinstance(python_trace, tuple) else f"{python_trace.engine}: {python_trace.message}" python_events: Final = python_trace if isinstance(python_trace, tuple) else () - rust_events: Final = rust_trace if isinstance(rust_trace, tuple) else () try: - python: Final = pipeline_projection("python", python_events) - rust: Final = pipeline_projection("rust", rust_events) + python: Final = pipeline_projection(python_events) except ValueError as error: return TraceArtifact.from_traces( - engine=effective_engine, surface=surface, sdk_function=route.route, scenario=scenario.name, python=(), - rust=(), python_error=f"harness: {error}", ) return TraceArtifact.from_traces( - engine=effective_engine, surface=surface, sdk_function=route.route, scenario=scenario.name, - python=python.steps, - rust=rust.steps, + python=python, python_error=python_error, - rust_error=rust_error, ) diff --git a/tests/rust-python-harness/strategies/trace_parity/sdk/messages/case.py b/tests/rust-python-harness/strategies/trace_parity/sdk/messages/case.py index 211c454eadf..4e6e50c7e37 100644 --- a/tests/rust-python-harness/strategies/trace_parity/sdk/messages/case.py +++ b/tests/rust-python-harness/strategies/trace_parity/sdk/messages/case.py @@ -2,7 +2,6 @@ from __future__ import annotations from typing import Final -from .....shared.tracing.steps import Engine, mapping from ...fixtures import ( anthropic_response_body, anthropic_stream_events, @@ -12,172 +11,44 @@ from ...fixtures import ( ) from ...models import RouteFixture, RouteSpec, TraceScenario, TraceSuite -COMMON_MAPPINGS: Final = ( - mapping(rust_span="messages", python_frame=r"anthropic_interface/messages/__init__\.py:\d+ a?create$"), - mapping(span="python_sanitize_empty_content", python_frame=r"strip_empty_content_blocks_from_anthropic_messages$"), - mapping(span="python_sanitize_tool_ids", python_frame=r"sanitize_tool_use_ids_in_anthropic_messages$"), - mapping( - span="python_flatten_web_search", python_frame=r"flatten_unencrypted_web_search_results_in_anthropic_messages$" - ), - mapping(span="python_cache_control", python_frame=r"AnthropicCacheControlHook\.maybe_inject_cache_control$"), - mapping(span="python_pre_request_hooks", python_frame=r"_execute_pre_request_hooks$"), - mapping( - span="python_messages_provider_config", - python_frame=r"ProviderConfigManager\.get_provider_anthropic_messages_config$", - ), - mapping(rust_span="messages_provider_config"), - mapping(rust_span="validate_environment", python_frame=r"validate_anthropic_messages_environment$"), - mapping(rust_span="complete_url", python_frame=r"get_complete_url$"), - mapping( - span="python_messages_entry_handler", - python_frame=r"messages/handler\.py:\d+ anthropic_messages_handler$", - ), - mapping( - span="python_messages_handler_wrapper", - python_frame=r"BaseLLMHTTPHandler\.anthropic_messages_handler$", - ), - mapping( - rust_span="execute_messages_provider_call", - python_frame=r"BaseLLMHTTPHandler\.async_anthropic_messages_handler$", - ), - mapping(rust_span="http_request", python_frame=r"AsyncHTTPHandler\.post$|HTTPHandler\.post$"), - mapping(rust_span="transform_response", python_frame=r"(? RouteFixture: +def _fixture(provider: str) -> RouteFixture: conversation: Final = {"messages": [{"role": "user", "content": "hello"}], "max_tokens": 16} return RouteFixture( kwargs={ "model": f"{provider}/claude-sonnet-5", - **({"body": {**conversation, "model": "claude-sonnet-5"}} if engine == "rust" else conversation), + **conversation, }, provider_responses=(json_response(anthropic_response_body()),), ) -def _anthropic_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _fixture(engine, "anthropic") +def _anthropic_fixture(_base_url: str) -> RouteFixture: + return _fixture("anthropic") -def _azure_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _fixture(engine, "azure_ai") +def _azure_fixture(_base_url: str) -> RouteFixture: + return _fixture("azure_ai") -def _bedrock_kwargs(engine: Engine) -> dict[str, object]: +def _bedrock_kwargs() -> dict[str, object]: conversation: Final = {"messages": [{"role": "user", "content": "hello"}], "max_tokens": 16} return { "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", - **( - {"body": {**conversation, "model": "anthropic.claude-3-sonnet-20240229-v1:0"}} - if engine == "rust" - else conversation - ), + **conversation, "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "aws_region_name": "us-east-1", } -def _bedrock_fixture(engine: Engine, _base_url: str) -> RouteFixture: - response_fixture: Final = _fixture(engine, "anthropic") - return RouteFixture(kwargs=_bedrock_kwargs(engine), provider_responses=response_fixture.provider_responses) +def _bedrock_fixture(_base_url: str) -> RouteFixture: + response_fixture: Final = _fixture("anthropic") + return RouteFixture(kwargs=_bedrock_kwargs(), provider_responses=response_fixture.provider_responses) -def _bedrock_retry_fixture(engine: Engine, _base_url: str) -> RouteFixture: - success_fixture: Final = _bedrock_fixture(engine, _base_url) +def _bedrock_retry_fixture(_base_url: str) -> RouteFixture: + success_fixture: Final = _bedrock_fixture(_base_url) messages: Final = [ {"role": "user", "content": "hello"}, { @@ -189,14 +60,7 @@ def _bedrock_retry_fixture(engine: Engine, _base_url: str) -> RouteFixture: }, {"role": "user", "content": "continue"}, ] - kwargs: Final = { - **_bedrock_kwargs(engine), - **( - {"body": {"messages": messages, "max_tokens": 16, "model": "anthropic.claude-3-sonnet-20240229-v1:0"}} - if engine == "rust" - else {"messages": messages} - ), - } + kwargs: Final = {**_bedrock_kwargs(), "messages": messages} return success_fixture.derive( kwargs=kwargs, provider_responses=( @@ -206,13 +70,13 @@ def _bedrock_retry_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _mock_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _fixture(engine, "anthropic") +def _mock_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _fixture("anthropic") return fixture.derive(kwargs={"mock_response": "hello from mock"}, provider_responses=()) -def _provider_error_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _fixture(engine, "anthropic") +def _provider_error_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _fixture("anthropic") return fixture.derive( provider_responses=( json_response( @@ -224,15 +88,13 @@ def _provider_error_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _sync_unsupported_fixture(engine: Engine, base_url: str) -> RouteFixture: - if engine == "rust": - return _anthropic_fixture(engine, base_url) - fixture: Final = _fixture(engine, "anthropic") +def _sync_unsupported_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _fixture("anthropic") return fixture.derive(provider_responses=(), expected_failure=True) -def _stream_fixture_for(engine: Engine, provider: str) -> RouteFixture: - fixture: Final = _fixture(engine, provider) +def _stream_fixture_for(provider: str) -> RouteFixture: + fixture: Final = _fixture(provider) return fixture.derive( kwargs={"stream": True}, provider_responses=(sse_response(anthropic_stream_events()),), @@ -240,16 +102,16 @@ def _stream_fixture_for(engine: Engine, provider: str) -> RouteFixture: ) -def _stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _stream_fixture_for(engine, "anthropic") +def _stream_fixture(_base_url: str) -> RouteFixture: + return _stream_fixture_for("anthropic") -def _azure_stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _stream_fixture_for(engine, "azure_ai") +def _azure_stream_fixture(_base_url: str) -> RouteFixture: + return _stream_fixture_for("azure_ai") -def _bedrock_stream_fixture(engine: Engine, base_url: str) -> RouteFixture: - fixture: Final = _bedrock_fixture(engine, base_url) +def _bedrock_stream_fixture(base_url: str) -> RouteFixture: + fixture: Final = _bedrock_fixture(base_url) events: Final = tuple(payload for _, payload in anthropic_stream_events()) return fixture.derive( kwargs={"stream": True}, @@ -258,8 +120,8 @@ def _bedrock_stream_fixture(engine: Engine, base_url: str) -> RouteFixture: ) -def _bedrock_stream_error_fixture(engine: Engine, base_url: str) -> RouteFixture: - fixture: Final = _bedrock_fixture(engine, base_url) +def _bedrock_stream_error_fixture(base_url: str) -> RouteFixture: + fixture: Final = _bedrock_fixture(base_url) start: Final = anthropic_stream_events(model="anthropic.claude-3-sonnet-20240229-v1:0")[0][1] return fixture.derive( kwargs={"stream": True}, @@ -269,56 +131,47 @@ def _bedrock_stream_error_fixture(engine: Engine, base_url: str) -> RouteFixture ) -SPEC: Final = RouteSpec("messages", ("create", "acreate"), ("messages", "amessages"), _anthropic_fixture) +SPEC: Final = RouteSpec("messages", ("create", "acreate"), _anthropic_fixture) TRACE_SUITE: Final = TraceSuite( route=SPEC, scenarios=( - TraceScenario( - name="async-anthropic", fixture=_anthropic_fixture, mappings=ANTHROPIC_MAPPINGS, asynchronous=True - ), - TraceScenario(name="async-azure-ai", fixture=_azure_fixture, mappings=AZURE_MAPPINGS, asynchronous=True), - TraceScenario(name="async-bedrock", fixture=_bedrock_fixture, mappings=BEDROCK_MAPPINGS, asynchronous=True), + TraceScenario(name="async-anthropic", fixture=_anthropic_fixture, asynchronous=True), + TraceScenario(name="async-azure-ai", fixture=_azure_fixture, asynchronous=True), + TraceScenario(name="async-bedrock", fixture=_bedrock_fixture, asynchronous=True), TraceScenario( name="async-bedrock-invalid-thinking-retry", fixture=_bedrock_retry_fixture, - mappings=RETRY_MAPPINGS, asynchronous=True, ), - TraceScenario(name="async-mock-response", fixture=_mock_fixture, mappings=MOCK_MAPPINGS, asynchronous=True), + TraceScenario(name="async-mock-response", fixture=_mock_fixture, asynchronous=True), TraceScenario( name="async-anthropic-provider-error", fixture=_provider_error_fixture, - mappings=ANTHROPIC_FAILURE_MAPPINGS, asynchronous=True, ), TraceScenario( name="async-anthropic-stream", fixture=_stream_fixture, - mappings=(*ANTHROPIC_MAPPINGS, *STREAM_MAPPINGS), asynchronous=True, ), TraceScenario( name="async-azure-ai-stream", fixture=_azure_stream_fixture, - mappings=(*AZURE_MAPPINGS, *STREAM_MAPPINGS), asynchronous=True, ), TraceScenario( name="async-bedrock-event-stream", fixture=_bedrock_stream_fixture, - mappings=(*BEDROCK_MAPPINGS, *STREAM_MAPPINGS), asynchronous=True, ), TraceScenario( name="async-bedrock-event-stream-error", fixture=_bedrock_stream_error_fixture, - mappings=(*BEDROCK_MAPPINGS, *STREAM_MAPPINGS, *FAILURE_MAPPINGS), asynchronous=True, ), TraceScenario( name="sync-unsupported", fixture=_sync_unsupported_fixture, - mappings=ANTHROPIC_MAPPINGS, asynchronous=False, ), ), diff --git a/tests/rust-python-harness/strategies/trace_parity/sdk/ocr/case.py b/tests/rust-python-harness/strategies/trace_parity/sdk/ocr/case.py index bb21e8ab0c5..036e6b48026 100644 --- a/tests/rust-python-harness/strategies/trace_parity/sdk/ocr/case.py +++ b/tests/rust-python-harness/strategies/trace_parity/sdk/ocr/case.py @@ -4,122 +4,10 @@ import json from typing import Final from .....shared.parity.recorded_http import HttpHeader, RecordedHttpResponse -from .....shared.tracing.steps import Engine, mapping from ...models import RouteFixture, RouteSpec, TraceScenario, TraceSuite -COMMON_MAPPINGS: Final = ( - mapping(rust_span="ocr", python_frame=r"ocr/main\.py:\d+ a?ocr$"), - mapping(rust_span="prepare_ocr_call", python_frame=r"ocr/main\.py:\d+ _prepare_ocr_request$"), - mapping(rust_span="ocr_provider_config", python_frame=r"ProviderConfigManager\.get_provider_ocr_config$"), - mapping(rust_span="supported_ocr_params", python_frame=r"get_supported_ocr_params$"), - mapping(rust_span="map_ocr_params", python_frame=r"(? RouteFixture: +def _fixture(model: str, document: dict[str, str] | None = None) -> RouteFixture: response: Final = json.dumps( { "pages": [{"index": 0, "markdown": "hello"}], @@ -131,7 +19,7 @@ def _fixture(engine: Engine, model: str, document: dict[str, str] | None = None) kwargs={ "model": model, "document": document or {"type": "document_url", "document_url": "https://example.com/document.pdf"}, - **({"optional_params": {"pages": [0]}} if engine == "rust" else {"pages": [0]}), + "pages": [0], }, provider_responses=( RecordedHttpResponse.from_bytes( @@ -141,12 +29,12 @@ def _fixture(engine: Engine, model: str, document: dict[str, str] | None = None) ) -def _mistral_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _fixture(engine, "mistral/mistral-ocr-latest") +def _mistral_fixture(_base_url: str) -> RouteFixture: + return _fixture("mistral/mistral-ocr-latest") -def _callback_fixture(engine: Engine, *, failure: bool) -> RouteFixture: - fixture: Final = _fixture(engine, "mistral/mistral-ocr-latest") +def _callback_fixture(*, failure: bool) -> RouteFixture: + fixture: Final = _fixture("mistral/mistral-ocr-latest") provider_responses: Final = ( ( RecordedHttpResponse.from_bytes( @@ -165,29 +53,28 @@ def _callback_fixture(engine: Engine, *, failure: bool) -> RouteFixture: ) -def _mistral_callback_success_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _callback_fixture(engine, failure=False) +def _mistral_callback_success_fixture(_base_url: str) -> RouteFixture: + return _callback_fixture(failure=False) -def _mistral_callback_failure_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _callback_fixture(engine, failure=True) +def _mistral_callback_failure_fixture(_base_url: str) -> RouteFixture: + return _callback_fixture(failure=True) -def _azure_fixture(engine: Engine, _base_url: str) -> RouteFixture: +def _azure_fixture(_base_url: str) -> RouteFixture: return _fixture( - engine, "azure_ai/pixtral-12b-2409", {"type": "image_url", "image_url": "data:image/png;base64,aGVsbG8="}, ) -def _vertex_deepseek_fixture(engine: Engine, _base_url: str) -> RouteFixture: +def _vertex_deepseek_fixture(_base_url: str) -> RouteFixture: vertex: Final = {"vertex_project": "trace-project", "vertex_location": "us-central1"} return RouteFixture( kwargs={ "model": "vertex_ai/deepseek-ocr-maas", "document": {"type": "image_url", "image_url": "data:image/png;base64,aGVsbG8="}, - **({"optional_params": vertex} if engine == "rust" else vertex), + **vertex, }, provider_responses=( RecordedHttpResponse.from_bytes( @@ -204,11 +91,11 @@ def _vertex_deepseek_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _vertex_deepseek_credentials_fixture(engine: Engine, base_url: str) -> RouteFixture: +def _vertex_deepseek_credentials_fixture(base_url: str) -> RouteFixture: from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa - fixture: Final = _vertex_deepseek_fixture(engine, base_url) + fixture: Final = _vertex_deepseek_fixture(base_url) private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) credentials: Final = json.dumps( { @@ -238,12 +125,12 @@ def _vertex_deepseek_credentials_fixture(engine: Engine, base_url: str) -> Route ) -def _cohere_fixture(engine: Engine, _base_url: str) -> RouteFixture: +def _cohere_fixture(_base_url: str) -> RouteFixture: return RouteFixture( kwargs={ "model": "cohere/parse-v5.0", "document": {"type": "image_url", "image_url": "data:image/png;base64,aGVsbG8="}, - **({"optional_params": {"output_format": "blocks"}} if engine == "rust" else {"output_format": "blocks"}), + "output_format": "blocks", }, provider_responses=( RecordedHttpResponse.from_bytes( @@ -260,7 +147,7 @@ def _cohere_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _azure_document_intelligence_fixture(engine: Engine, base_url: str) -> RouteFixture: +def _azure_document_intelligence_fixture(base_url: str) -> RouteFixture: completed: Final = json.dumps( { "status": "succeeded", @@ -285,7 +172,7 @@ def _azure_document_intelligence_fixture(engine: Engine, base_url: str) -> Route "type": "document_url", "document_url": "data:application/pdf;base64,aGVsbG8=", }, - **({"optional_params": {"pages": [0]}} if engine == "rust" else {"pages": [0]}), + "pages": [0], }, provider_responses=( RecordedHttpResponse.from_bytes( @@ -305,204 +192,83 @@ def _azure_document_intelligence_fixture(engine: Engine, base_url: str) -> Route ) -DEEPSEEK_COMMON_MAPPINGS: Final = ( - mapping(rust_span="ocr", python_frame=r"ocr/main\.py:\d+ a?ocr$"), - mapping(rust_span="prepare_ocr_call", python_frame=r"ocr/main\.py:\d+ _prepare_ocr_request$"), - mapping(rust_span="ocr_provider_config", python_frame=r"ProviderConfigManager\.get_provider_ocr_config$"), - mapping(rust_span="supported_ocr_params", python_frame=r"get_supported_ocr_params$"), - mapping(rust_span="map_ocr_params", python_frame=r"(? RouteFixture: +def _native_fixture(provider: str) -> RouteFixture: model: Final = "gpt-5" return RouteFixture( kwargs={ "model": f"{provider}/{model}", "input": "hello", - **({"body": {"model": model, "input": "hello"}} if engine == "rust" else {}), }, provider_responses=(json_response(responses_body(model=model)),), ) -def _openai_fixture(engine: Engine, _base_url: str) -> RouteFixture: - return _native_fixture(engine, "openai") +def _openai_fixture(_base_url: str) -> RouteFixture: + return _native_fixture("openai") -def _azure_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _native_fixture(engine, "azure") +def _azure_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _native_fixture("azure") return fixture.derive(kwargs={"api_version": "2025-04-01-preview"}) -def _openai_stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _openai_fixture(engine, _base_url) +def _openai_stream_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _openai_fixture(_base_url) return fixture.derive( kwargs={"stream": True}, provider_responses=(sse_response(responses_stream_events()),), @@ -107,8 +42,8 @@ def _openai_stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _provider_error_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _openai_fixture(engine, _base_url) +def _provider_error_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _openai_fixture(_base_url) return fixture.derive( provider_responses=( json_response({"error": {"message": "bad request", "type": "invalid_request_error"}}, status=400), @@ -117,8 +52,8 @@ def _provider_error_fixture(engine: Engine, _base_url: str) -> RouteFixture: ) -def _stream_failed_fixture(engine: Engine, base_url: str) -> RouteFixture: - fixture: Final = _openai_fixture(engine, base_url) +def _stream_failed_fixture(base_url: str) -> RouteFixture: + fixture: Final = _openai_fixture(base_url) failed_response: Final[dict[str, object]] = { **responses_body(), "status": "failed", @@ -140,20 +75,19 @@ def _stream_failed_fixture(engine: Engine, base_url: str) -> RouteFixture: ) -def _anthropic_bridge_fixture(engine: Engine, _base_url: str) -> RouteFixture: +def _anthropic_bridge_fixture(_base_url: str) -> RouteFixture: return RouteFixture( kwargs={ "model": "anthropic/claude-sonnet-5", "input": "hello", "max_output_tokens": 16, - **({"body": {"model": "claude-sonnet-5", "input": "hello"}} if engine == "rust" else {}), }, provider_responses=(json_response(anthropic_response_body()),), ) -def _anthropic_bridge_stream_fixture(engine: Engine, _base_url: str) -> RouteFixture: - fixture: Final = _anthropic_bridge_fixture(engine, _base_url) +def _anthropic_bridge_stream_fixture(_base_url: str) -> RouteFixture: + fixture: Final = _anthropic_bridge_fixture(_base_url) return fixture.derive( kwargs={"stream": True}, provider_responses=(sse_response(anthropic_stream_events()),), @@ -161,55 +95,41 @@ def _anthropic_bridge_stream_fixture(engine: Engine, _base_url: str) -> RouteFix ) -SPEC: Final = RouteSpec("responses", ("responses", "aresponses"), None, _openai_fixture) +SPEC: Final = RouteSpec("responses", ("responses", "aresponses"), _openai_fixture) TRACE_SUITE: Final = TraceSuite( route=SPEC, scenarios=( - TraceScenario(name="sync-openai", fixture=_openai_fixture, mappings=COMMON_MAPPINGS, asynchronous=False), - TraceScenario(name="async-openai", fixture=_openai_fixture, mappings=COMMON_MAPPINGS, asynchronous=True), + TraceScenario(name="sync-openai", fixture=_openai_fixture, asynchronous=False), + TraceScenario(name="async-openai", fixture=_openai_fixture, asynchronous=True), TraceScenario( name="sync-openai-stream", fixture=_openai_stream_fixture, - mappings=(*COMMON_MAPPINGS, *STREAM_MAPPINGS), asynchronous=False, ), TraceScenario( name="async-openai-stream", fixture=_openai_stream_fixture, - mappings=(*COMMON_MAPPINGS, *STREAM_MAPPINGS), asynchronous=True, ), TraceScenario( name="async-openai-provider-error", fixture=_provider_error_fixture, - mappings=(*COMMON_MAPPINGS, *FAILURE_MAPPINGS), asynchronous=True, ), TraceScenario( name="async-openai-stream-failed", fixture=_stream_failed_fixture, - mappings=(*COMMON_MAPPINGS, *STREAM_MAPPINGS, *FAILURE_MAPPINGS), asynchronous=True, ), - TraceScenario(name="async-azure", fixture=_azure_fixture, mappings=AZURE_MAPPINGS, asynchronous=True), + TraceScenario(name="async-azure", fixture=_azure_fixture, asynchronous=True), TraceScenario( name="async-anthropic-chat-bridge", fixture=_anthropic_bridge_fixture, - mappings=BRIDGE_MAPPINGS, asynchronous=True, ), TraceScenario( name="async-anthropic-chat-bridge-stream", fixture=_anthropic_bridge_stream_fixture, - mappings=( - *BRIDGE_MAPPINGS, - mapping(span="python_chat_stream_wrapper", python_frame=r"CustomStreamWrapper\.__init__$"), - mapping(span="python_chat_stream_next", python_frame=r"CustomStreamWrapper\.__anext__$"), - mapping( - span="python_responses_bridge_stream_iterator", - python_frame=r"LiteLLMCompletionStreamingIterator\.__init__$|LiteLLMCompletionStreamingIterator\.__anext__$", - ), - ), asynchronous=True, ), ), diff --git a/tests/rust-python-harness/strategies/trace_parity/sdk/test_core_scenario_matrix.py b/tests/rust-python-harness/strategies/trace_parity/sdk/test_core_scenario_matrix.py index d0dbd281a97..47c5948af75 100644 --- a/tests/rust-python-harness/strategies/trace_parity/sdk/test_core_scenario_matrix.py +++ b/tests/rust-python-harness/strategies/trace_parity/sdk/test_core_scenario_matrix.py @@ -41,8 +41,8 @@ def test_core_sdk_scenario_matrix_keeps_distinct_migration_paths() -> None: } assert {(scenario.name, scenario.asynchronous) for scenario in ocr.scenarios} >= { ("async-cohere", True), - ("sync-public-rust-dispatch", False), - ("async-public-rust-dispatch", True), + ("sync-vertex-deepseek", False), + ("async-vertex-deepseek", True), } assert {(scenario.name, scenario.asynchronous) for scenario in responses.scenarios} >= { ("sync-openai", False), @@ -55,15 +55,3 @@ def test_core_sdk_scenario_matrix_keeps_distinct_migration_paths() -> None: ("async-anthropic-chat-bridge", True), ("async-anthropic-chat-bridge-stream", True), } - - -def test_core_gateway_matrix_keeps_downstream_streams_separate() -> None: - modules: Final = ( - "tests.rust-python-harness.strategies.trace_parity.gateway.chat_completions.case", - "tests.rust-python-harness.strategies.trace_parity.gateway.messages.case", - "tests.rust-python-harness.strategies.trace_parity.gateway.responses.case", - ) - - for module in modules: - suite = _suite(module) - assert any("downstream-stream" in scenario.name for scenario in suite.scenarios) diff --git a/tests/rust-python-harness/strategies/trace_parity/sdk/transcription/case.py b/tests/rust-python-harness/strategies/trace_parity/sdk/transcription/case.py index 3b4d2e1447d..2071e00d3d6 100644 --- a/tests/rust-python-harness/strategies/trace_parity/sdk/transcription/case.py +++ b/tests/rust-python-harness/strategies/trace_parity/sdk/transcription/case.py @@ -7,41 +7,8 @@ import wave from typing import Final from .....shared.parity.recorded_http import HttpHeader, RecordedHttpResponse -from .....shared.tracing.steps import Engine, mapping from ...models import RouteFixture, RouteSpec, TraceScenario, TraceSuite -MAPPINGS: Final = ( - mapping(rust_span="prepare_audio_transcription_provider_call"), - mapping(span="get_non_default_params", python_frame=r"get_non_default_transcription_params$"), - mapping(rust_span="map_transcription_params", python_frame=r"get_optional_params_transcription$"), - mapping( - span="python_provider_config", - python_frame=r"ProviderConfigManager\.get_provider_audio_transcription_config$", - ), - mapping(rust_span="provider_config"), - mapping(rust_span="supported_transcription_params"), - mapping(rust_span="transform_transcription_request"), - mapping( - rust_span="execute_audio_transcription_provider_call", - python_frame=r"BedrockAudioTranscriptionRustDispatch\.(?:async_)?audio_transcriptions$", - ), - mapping(rust_span="transform_transcription_response"), - mapping(rust_span="http_request"), -) - -SYNC_MAPPINGS: Final = ( - mapping(rust_span="audio_transcription", python_frame=r"main\.py:\d+ transcription$"), - *MAPPINGS, -) -ASYNC_MAPPINGS: Final = ( - mapping(rust_span="audio_transcription", python_frame=r"main\.py:\d+ atranscription$"), - mapping(span="python_transcription_wrapper", python_frame=r"main\.py:\d+ transcription$"), - *MAPPINGS[:2], - mapping(span="python_map_transcription_params", python_frame=r"get_optional_params_transcription$"), - mapping(rust_span="map_transcription_params"), - *MAPPINGS[3:], -) - def _audio_bytes() -> bytes: with io.BytesIO() as buffer: @@ -53,18 +20,14 @@ def _audio_bytes() -> bytes: return buffer.getvalue() -def _fixture(engine: Engine, _base_url: str) -> RouteFixture: +def _fixture(_base_url: str) -> RouteFixture: credentials: Final = { "aws_access_key_id": "test-access", "aws_secret_access_key": "test-secret", "aws_region_name": "us-east-1", } audio: Final = _audio_bytes() - payload: Final = ( - {"audio": {"data": base64.b64encode(audio).decode(), "format": "wav"}, "optional_params": credentials} - if engine == "rust" - else {"file": ("sample.wav", audio, "audio/wav"), **credentials} - ) + payload: Final = {"file": ("sample.wav", audio, "audio/wav"), **credentials} response: Final = json.dumps( { "output": {"message": {"role": "assistant", "content": [{"text": "hello"}]}}, @@ -85,7 +48,6 @@ def _fixture(engine: Engine, _base_url: str) -> RouteFixture: SPEC: Final = RouteSpec( "transcription", ("transcription", "atranscription"), - ("transcription", "atranscription"), _fixture, ) TRACE_SUITE: Final = TraceSuite( @@ -94,13 +56,11 @@ TRACE_SUITE: Final = TraceSuite( TraceScenario( name="sync-bedrock", fixture=_fixture, - mappings=SYNC_MAPPINGS, asynchronous=False, ), TraceScenario( name="async-bedrock", fixture=_fixture, - mappings=ASYNC_MAPPINGS, asynchronous=True, ), ), diff --git a/tests/rust-python-harness/strategies/trace_parity/test_reporting.py b/tests/rust-python-harness/strategies/trace_parity/test_reporting.py index 22cc87592b8..2d5ed14b6cd 100644 --- a/tests/rust-python-harness/strategies/trace_parity/test_reporting.py +++ b/tests/rust-python-harness/strategies/trace_parity/test_reporting.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Final, Literal +from typing import Final import pytest @@ -28,20 +28,16 @@ def _result(trace: TraceArtifact) -> CaseResult: def _trace( python: tuple[PipelineStep, ...], - rust: tuple[PipelineStep, ...], *, - rust_error: str | None = None, - engine: Literal["python", "rust", "both"] = "both", + python_error: str | None = None, scenario: str = "sync-default", ) -> TraceArtifact: return TraceArtifact.from_traces( - engine=engine, surface="sdk", sdk_function="ocr", scenario=scenario, python=python, - rust=rust, - rust_error=rust_error, + python_error=python_error, ) @@ -55,52 +51,29 @@ def _events(*items: tuple[str, int, str | None]) -> tuple[PipelineStep, ...]: return tuple(steps) -def test_renderer_prints_python_and_rust_traces_independently() -> None: +def test_renderer_prints_the_python_trace() -> None: python: Final = _events( ("ocr", 0, "ocr/main.py:88 aocr"), ("python_prepare", 1, "prep.py:1 python_prepare"), ) - rust: Final = _events(("ocr", 0, None), ("rust_prepare", 1, None)) - section: Final = render_trace_results((_result(_trace(python, rust)),))[0] + section: Final = render_trace_results((_result(_trace(python)),))[0] report: Final = "\n\n".join(section.blocks) assert section.title == "SDK traces" assert "PYTHON (2 steps)\n1 aocr (ocr/main.py:88)\n2 python_prepare (prep.py:1)" in report - assert "RUST (2 steps)\n1 ocr\n2 rust_prepare" in report - assert "python only" not in report - assert "rust only" not in report - assert " -> " not in report - assert "Trace: MATCH" not in report - assert "Trace: DRIFT" not in report - assert "Contract:" not in report + assert "RUST" not in report -@pytest.mark.parametrize( - ("engine", "present", "absent"), - (("python", "PYTHON (1 steps)", "RUST"), ("rust", "RUST (1 steps)", "PYTHON")), -) -def test_renderer_prints_only_selected_engine(engine: Literal["python", "rust"], present: str, absent: str) -> None: - events: Final = _events(("ocr", 0, None)) - - report: Final = "\n\n".join(render_trace_results((_result(_trace(events, events, engine=engine)),))[0].blocks) - - assert present in report - assert absent not in report - - -def test_renderer_keeps_collected_trace_when_one_engine_errors() -> None: +def test_renderer_keeps_collected_trace_when_python_errors() -> None: python: Final = _events(("ocr", 0, "ocr/main.py:88 aocr")) report: Final = "\n\n".join( - render_trace_results( - (_result(_trace(python, (), rust_error="rust: native Rust bridge must include the trace-parity feature")),) - )[0].blocks + render_trace_results((_result(_trace(python, python_error="python: replay server closed")),))[0].blocks ) assert "PYTHON (1 steps)\n1 aocr (ocr/main.py:88)" in report - assert "Rust error: rust: native Rust bridge must include the trace-parity feature" in report - assert "hint: rebuild the native bridge with the trace-parity feature" in report + assert "Python error: python: replay server closed" in report def test_unavailable_trace_reports_scenario_from_nodeid() -> None: @@ -122,8 +95,8 @@ def test_unavailable_trace_reports_scenario_from_nodeid() -> None: def test_renderer_groups_scenarios_under_one_case_header() -> None: - result: Final = _result(_trace(_events(("ocr", 0, None)), (), scenario="sync-default")) - async_trace: Final = _trace((), _events(("ocr", 0, None)), scenario="async-default") + result: Final = _result(_trace(_events(("ocr", 0, None)), scenario="sync-default")) + async_trace: Final = _trace(_events(("ocr", 0, None)), scenario="async-default") nodeid: Final = "trace:sdk:ocr:async-default" result.collected.add(nodeid) result.record(nodeid, RunStatus.PASSED, artifacts=(ResultArtifact(TRACE_ARTIFACT, async_trace.model_dump_json()),)) @@ -140,12 +113,10 @@ def test_renderer_colors_every_trace_line_in_a_terminal(monkeypatch: pytest.Monk monkeypatch.setattr(reporting.sys.stdout, "isatty", lambda: True) monkeypatch.delenv("NO_COLOR", raising=False) - report: Final = "\n\n".join(render_trace_results((_result(_trace(events, events)),))[0].blocks) + report: Final = "\n\n".join(render_trace_results((_result(_trace(events)),))[0].blocks) assert "\033[36mPYTHON\033[0m (1 steps)" in report assert "\033[36m1 aocr (ocr/main.py:88)\033[0m" in report - assert "\033[33mRUST\033[0m (1 steps)" in report - assert "\033[33m1 ocr\033[0m" in report def test_renderer_groups_unavailable_entries_by_surface() -> None: @@ -160,7 +131,7 @@ def test_renderer_groups_unavailable_entries_by_surface() -> None: status=RunStatus.NOT_IMPLEMENTED, ) - sections: Final = render_trace_results((_result(_trace((), ())), gateway_result)) + sections: Final = render_trace_results((_result(_trace(())), gateway_result)) assert tuple(section.title for section in sections) == ("SDK traces", "GATEWAY traces") assert "- messages: No messages case is registered." in "\n\n".join(sections[1].blocks) diff --git a/tests/rust-python-harness/strategies/trace_parity/test_runner.py b/tests/rust-python-harness/strategies/trace_parity/test_runner.py index be25dd53b02..a5b66ab0088 100644 --- a/tests/rust-python-harness/strategies/trace_parity/test_runner.py +++ b/tests/rust-python-harness/strategies/trace_parity/test_runner.py @@ -13,14 +13,14 @@ import litellm from ...shared.reporting.models import Coverage, HarnessCase, HarnessRun, RunStatus, SdkFunction, Surface from ...shared.reporting.strategy import ModuleCaseSpec from ...shared.tracing.profiler import FunctionTraceEvent -from ...shared.tracing.steps import Engine, PipelineStep, mapping -from .models import GatewayRouteSpec, RouteFixture, RouteSpec, TraceScenario, TraceSuite +from ...shared.tracing.steps import PipelineStep +from .models import RouteFixture, RouteSpec, TraceScenario, TraceSuite from .reporting import TraceArtifact -from .runner import run_trace_cases, run_trace_scenario, runner_selection, scenario_nodeids, validate_trace_suite +from .runner import run_trace_cases, run_trace_scenario, scenario_nodeids, validate_trace_suite from .sdk.execution import SdkCall, collect_trace, execute_trace -def _fixture(_engine: Engine, _base_url: str) -> RouteFixture: +def _fixture(_base_url: str) -> RouteFixture: return RouteFixture(kwargs={}, provider_responses=()) @@ -36,11 +36,11 @@ def _case(*, surface: Surface = "sdk", function: SdkFunction = "ocr") -> Harness def test_scenario_filtering_and_occurrence_node_ids() -> None: suite: Final = TraceSuite( - route=RouteSpec("ocr", ("ocr", "aocr"), ("ocr", "aocr"), _fixture), + route=RouteSpec("ocr", ("ocr", "aocr"), _fixture), scenarios=( - TraceScenario("sync-one", _fixture, (), asynchronous=False), - TraceScenario("async-one", _fixture, (), asynchronous=True), - TraceScenario("async-two", _fixture, (), asynchronous=True), + TraceScenario("sync-one", _fixture, asynchronous=False), + TraceScenario("async-one", _fixture, asynchronous=True), + TraceScenario("async-two", _fixture, asynchronous=True), ), ) case: Final = _case() @@ -50,45 +50,35 @@ def test_scenario_filtering_and_occurrence_node_ids() -> None: assert tuple(nodeid for _, nodeid in nodes) == ("trace:sdk:ocr:async-two",) -def test_python_engine_is_separate_from_scenario_selection() -> None: - assert runner_selection(("mistral", "--engine=python")) == (frozenset({"mistral"}), "python") - - -def test_python_engine_skips_native_bridge(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: +def test_runner_arguments_select_scenarios(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: runner: Final = importlib.import_module("tests.rust-python-harness.strategies.trace_parity.runner") case: Final = _case() - selected: list[tuple[frozenset[str], str]] = [] - - def reject_bridge(_repo_root: Path) -> str | None: - raise AssertionError("Python-only tracing must not inspect or build the native bridge") + selected: list[frozenset[str]] = [] def capture_case( _run: HarnessRun, _case: HarnessCase, scenarios: frozenset[str], _on_update: object, - engine: str, ) -> None: - selected.append((scenarios, engine)) + selected.append(scenarios) - monkeypatch.setattr(runner, "ensure_trace_bridge", reject_bridge) monkeypatch.setattr(runner, "_run_case", capture_case) - exit_code, _ = run_trace_cases((case,), tmp_path, lambda _: None, ("mistral", "--engine=python")) + exit_code, _ = run_trace_cases((case,), tmp_path, lambda _: None, ("mistral",)) assert exit_code == 0 - assert selected == [(frozenset({"mistral"}), "python")] + assert selected == [frozenset({"mistral"})] def test_python_trace_preserves_native_ocr_dispatch_setting(monkeypatch: pytest.MonkeyPatch) -> None: execution: Final = importlib.import_module("tests.rust-python-harness.strategies.trace_parity.sdk.execution") - route: Final = RouteSpec("ocr", ("ocr", "aocr"), ("ocr", "aocr"), _fixture) + route: Final = RouteSpec("ocr", ("ocr", "aocr"), _fixture) observed: list[str | None] = [] def collect( _function: SdkCall, _fixture: RouteFixture, - _engine: Engine, *, asynchronous: bool, ) -> SimpleNamespace: @@ -101,9 +91,9 @@ def test_python_trace_preserves_native_ocr_dispatch_setting(monkeypatch: pytest. monkeypatch.setattr(execution, "_collect", collect) monkeypatch.setenv("LITELLM_RUST", "0") - collect_trace(route, "python", asynchronous=False) + collect_trace(route, asynchronous=False) monkeypatch.setenv("LITELLM_RUST", "1") - collect_trace(route, "python", asynchronous=True) + collect_trace(route, asynchronous=True) assert observed == ["0", "1"] assert os.environ["LITELLM_RUST"] == "1" @@ -116,9 +106,8 @@ def test_expected_provider_failure_omits_feedback_banner( suite: Final = cast(TraceSuite, loaded.TRACE_SUITE) scenario: Final = next(item for item in suite.scenarios if item.name == "async-openai-provider-error") monkeypatch.setattr(litellm, "suppress_debug_info", False) - assert isinstance(suite.route, RouteSpec) - result: Final = execute_trace(suite.route, scenario, "sdk", engine="python") + result: Final = execute_trace(suite.route, scenario, "sdk") assert result.python_error is None assert "Give Feedback / Get Help" not in capsys.readouterr().out @@ -131,9 +120,8 @@ def test_vertex_trace_keeps_unmapped_helpers_and_parents(asynchronous: bool) -> suite: Final = cast(TraceSuite, loaded.TRACE_SUITE) name: Final = f"{'async' if asynchronous else 'sync'}-vertex-deepseek" scenario: Final = next(item for item in suite.scenarios if item.name == name) - assert isinstance(suite.route, RouteSpec) - trace: Final = execute_trace(suite.route, scenario, "sdk", engine="python") + trace: Final = execute_trace(suite.route, scenario, "sdk") assert trace.python_error is None url: Final = next( @@ -157,9 +145,8 @@ def test_vertex_credentials_trace_runs_real_auth_helpers(asynchronous: bool, mon scenario: Final = next(item for item in suite.scenarios if item.name == name) monkeypatch.setenv("VERTEXAI_CREDENTIALS", "original-credentials") monkeypatch.setenv("VERTEX_AI_API_KEY", "original-api-key") - assert isinstance(suite.route, RouteSpec) - trace: Final = execute_trace(suite.route, scenario, "sdk", engine="python") + trace: Final = execute_trace(suite.route, scenario, "sdk") assert trace.python_error is None validate: Final = next( @@ -180,79 +167,16 @@ def test_vertex_credentials_trace_runs_real_auth_helpers(asynchronous: bool, mon assert os.environ["VERTEX_AI_API_KEY"] == "original-api-key" -def test_gateway_trace_keeps_calls_outside_scenario_mappings(monkeypatch: pytest.MonkeyPatch) -> None: - execution: Final = importlib.import_module("tests.rust-python-harness.strategies.trace_parity.gateway.execution") - events: Final = ( - FunctionTraceEvent(0, None, "route.py:1 entry"), - FunctionTraceEvent(1, 0, "auth.py:2 authenticate"), - FunctionTraceEvent(2, 1, "auth.py:3 credentials"), - ) - scenario: Final = TraceScenario( - "async-gateway", - _fixture, - (mapping(rust_span="entry", python_frame=r" entry$"),), - asynchronous=True, - ) - monkeypatch.setattr(execution, "_collect", lambda *_args: events) - - trace: Final = execution.execute_gateway_trace(GatewayRouteSpec("messages"), scenario, engine="python") - - assert trace.python_error is None - assert tuple((event.id, event.parent_id, event.raw) for event in trace.python) == tuple( - (event.id, event.parent_id, event.raw) for event in events - ) - - -def test_default_trace_skips_unavailable_rust_sdk_entrypoint(monkeypatch: pytest.MonkeyPatch) -> None: - execution: Final = importlib.import_module("tests.rust-python-harness.strategies.trace_parity.sdk.execution") - route: Final = RouteSpec("responses", ("responses", "aresponses"), None, _fixture) - scenario: Final = TraceScenario("sync-openai", _fixture, (), asynchronous=False) - engines: list[Engine] = [] - - def collect(_route: RouteSpec, engine: Engine, *, asynchronous: bool) -> tuple[FunctionTraceEvent, ...]: - engines.append(engine) - return (FunctionTraceEvent(0, None, "responses"),) - - monkeypatch.setattr(execution, "collect_trace", collect) - - trace: Final = execution.execute_trace(route, scenario, "sdk") - - assert engines == ["python"] - assert trace.engine == "python" - assert trace.rust_error is None - - -def test_default_trace_skips_unavailable_rust_gateway_route(monkeypatch: pytest.MonkeyPatch) -> None: - execution: Final = importlib.import_module("tests.rust-python-harness.strategies.trace_parity.gateway.execution") - route: Final = GatewayRouteSpec("responses", rust_supported=False) - scenario: Final = TraceScenario("async-openai", _fixture, (), asynchronous=True) - engines: list[Engine] = [] - - def collect(_route: GatewayRouteSpec, _scenario: TraceScenario, engine: Engine) -> tuple[FunctionTraceEvent, ...]: - engines.append(engine) - return (FunctionTraceEvent(0, None, "responses"),) - - monkeypatch.setattr(execution, "_collect", collect) - - trace: Final = execution.execute_gateway_trace(route, scenario) - - assert engines == ["python"] - assert trace.engine == "python" - assert trace.rust_error is None - - def test_scenario_validation_rejects_duplicate_and_unsafe_names() -> None: - route: Final = RouteSpec("ocr", ("ocr", "aocr"), ("ocr", "aocr"), _fixture) + route: Final = RouteSpec("ocr", ("ocr", "aocr"), _fixture) duplicate: Final = TraceSuite( route=route, scenarios=( - TraceScenario("sync-same", _fixture, (), asynchronous=False), - TraceScenario("sync-same", _fixture, (), asynchronous=False), + TraceScenario("sync-same", _fixture, asynchronous=False), + TraceScenario("sync-same", _fixture, asynchronous=False), ), ) - unsafe: Final = TraceSuite( - route=route, scenarios=(TraceScenario("sync-bad:name", _fixture, (), asynchronous=False),) - ) + unsafe: Final = TraceSuite(route=route, scenarios=(TraceScenario("sync-bad:name", _fixture, asynchronous=False),)) case: Final = _case() assert validate_trace_suite(duplicate, case) is not None @@ -261,22 +185,22 @@ def test_scenario_validation_rejects_duplicate_and_unsafe_names() -> None: def test_scenario_validation_rejects_invalid_names_and_route_registration() -> None: invalid_name: Final = TraceSuite( - route=RouteSpec("ocr", ("ocr", "aocr"), ("ocr", "aocr"), _fixture), - scenarios=(TraceScenario("bedrock", _fixture, (), asynchronous=True),), + route=RouteSpec("ocr", ("ocr", "aocr"), _fixture), + scenarios=(TraceScenario("bedrock", _fixture, asynchronous=True),), ) wrong_function: Final = TraceSuite( - route=RouteSpec("messages", ("create", "acreate"), ("messages", "amessages"), _fixture), - scenarios=(TraceScenario("sync-one", _fixture, (), asynchronous=False),), + route=RouteSpec("messages", ("create", "acreate"), _fixture), + scenarios=(TraceScenario("sync-one", _fixture, asynchronous=False),), ) wrong_surface: Final = TraceSuite( - route=GatewayRouteSpec("ocr"), - scenarios=(TraceScenario("sync-one", _fixture, (), asynchronous=False),), + route=RouteSpec("ocr", ("ocr", "aocr"), _fixture), + scenarios=(TraceScenario("sync-one", _fixture, asynchronous=False),), ) case: Final = _case() assert "start with sync- or async-" in (validate_trace_suite(invalid_name, case) or "") assert "does not match case function" in (validate_trace_suite(wrong_function, case) or "") - assert "must use RouteSpec" in (validate_trace_suite(wrong_surface, case) or "") + assert "requires the sdk surface" in (validate_trace_suite(wrong_surface, _case(surface="gateway")) or "") def test_invalid_route_dispatch_records_harness_error() -> None: @@ -284,32 +208,31 @@ def test_invalid_route_dispatch_records_harness_error() -> None: run: Final = HarnessRun.from_cases((case,)) result: Final = run.results[case.key] suite: Final = TraceSuite( - route=GatewayRouteSpec("ocr"), - scenarios=(TraceScenario("sync-one", _fixture, (), asynchronous=False),), + route=RouteSpec("ocr", ("ocr", "aocr"), _fixture), + scenarios=(TraceScenario("sync-one", _fixture, asynchronous=False),), ) - nodeid: Final = "trace:sdk:ocr:sync-one" + nodeid: Final = "trace:gateway:ocr:sync-one" - run_trace_scenario(run, result, suite, suite.scenarios[0], "sdk", nodeid, lambda _: None) + run_trace_scenario(run, result, suite, suite.scenarios[0], "gateway", nodeid, lambda _: None) assert result.outcomes[nodeid] is RunStatus.ERROR - assert run.failures == [(nodeid, "gateway route cannot run on the sdk surface")] + assert run.failures == [(nodeid, "trace scenarios only run on the sdk surface")] -def test_different_python_and_rust_traces_pass(monkeypatch: pytest.MonkeyPatch) -> None: +def test_python_trace_without_errors_passes(monkeypatch: pytest.MonkeyPatch) -> None: runner: Final = importlib.import_module("tests.rust-python-harness.strategies.trace_parity.runner") case: Final = _case() run: Final = HarnessRun.from_cases((case,)) result: Final = run.results[case.key] suite: Final = TraceSuite( - route=RouteSpec("ocr", ("ocr", "aocr"), ("ocr", "aocr"), _fixture), - scenarios=(TraceScenario("sync-one", _fixture, (), asynchronous=False),), + route=RouteSpec("ocr", ("ocr", "aocr"), _fixture), + scenarios=(TraceScenario("sync-one", _fixture, asynchronous=False),), ) trace: Final = TraceArtifact.from_traces( surface="sdk", sdk_function="ocr", scenario="sync-one", python=(PipelineStep(0, None, "python_step", "python.py:1 python_step"),), - rust=(PipelineStep(0, None, "rust_step", "rust_step"),), ) monkeypatch.setattr(runner, "_execute_scenario", lambda *_args: trace) diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/AGENTS.md b/tests/rust-python-harness/strategies/unit_tests_mapping/AGENTS.md deleted file mode 100644 index 379d1443f33..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/AGENTS.md +++ /dev/null @@ -1,13 +0,0 @@ -# What this is - -Validates that unit tests covering traced Python behavior have semantic counterparts among colocated Rust unit tests - -# How it works - -Trace parity runs representative public API scenarios and records the Python and Rust functions reached, including their source files and lines. The OCR contract selects the behavior-level trace spans that require parity and excludes shared infrastructure such as generic HTTP transport - -For Python, those traced functions define the denominator. Static references and explicit includes create a safe pytest discovery universe, then a pytest profiler keeps only tests that actually execute at least one selected function. Static matches do not count by themselves. Parametrized pytest cases are collapsed to one logical test function in the mapping report. Explicit includes and exclusions cover dynamic callers or intentional harness behavior that static discovery cannot express reliably - -For Rust, each traced function identifies its source file and module. If that source file has a colocated `#[cfg(test)] mod tests`, the harness inventories that module for the configured Rust target. Rust test names are therefore derived from traced implementation files, not from a hand-maintained list of OCR test modules - -The Python-to-Rust mappings remain explicit because equivalent behavior often has different test boundaries and names in each SDK. Host-only exclusions require a reason. The report validates both against the live inventories, then shows mapped, excluded, and unmapped Python tests plus Rust-only tests diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/__init__.py b/tests/rust-python-harness/strategies/unit_tests_mapping/__init__.py deleted file mode 100644 index 4d857c01ed0..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import annotations - -from functools import partial -from pathlib import Path -from typing import Final - -from ...shared.reporting.models import SDK_FUNCTIONS, Coverage -from ...shared.reporting.strategy import ( - CaseDefinition, - NotImplementedCaseSpec, - RunnerArgumentDefinition, - StrategyDefinition, - SuiteCaseSpec, -) -from ...shared.unit_runners.suite_runner import run_suites -from .mappings import UNIT_TEST_CONTRACTS -from .reporting import render_mapping_results -from .runner import run_suite - - -CASES: Final[tuple[CaseDefinition, ...]] = ( - *( - CaseDefinition( - sdk_function, - SuiteCaseSpec(coverage=Coverage.COMPLETE, suite=sdk_function) - if sdk_function in UNIT_TEST_CONTRACTS - else NotImplementedCaseSpec(reason=f"No {sdk_function} unit-test mapping is registered."), - ) - for sdk_function in SDK_FUNCTIONS - ), -) - -STRATEGY: Final = StrategyDefinition( - id="unit_tests_mapping", - order=30, - label="Unit test mapping", - description="Validate Python/Rust unit-test mappings against collected test inventories.", - directory=Path(__file__).parent, - runnable_spec=SuiteCaseSpec, - cases=CASES, - run=partial(run_suites, suites=UNIT_TEST_CONTRACTS, execute=run_suite), - render=render_mapping_results, - runner_argument=RunnerArgumentDefinition( - option="--detail", - metavar="MODE", - help="show individual test names; any value enables full detail", - ), -) diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/cases/__init__.py b/tests/rust-python-harness/strategies/unit_tests_mapping/cases/__init__.py deleted file mode 100644 index 8b137891791..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/cases/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/cases/ocr.py b/tests/rust-python-harness/strategies/unit_tests_mapping/cases/ocr.py deleted file mode 100644 index 0e771f0dc17..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/cases/ocr.py +++ /dev/null @@ -1,422 +0,0 @@ -from __future__ import annotations - -from typing import Final - -from ....shared.unit_runners.rust_runner import RustTarget, RustTestIdentity -from ..contracts import ( - MappingExclusionSpec, - MappingSpec, - PythonFunctionDiscoverySpec, - RustTestFamily, - RustUnitSpec, - TestMapping, - UnitParityExclusionSpec, - UnitParitySpec, - UnitTestContract, -) - -_CORE_TARGET: Final = RustTarget(package="litellm-core", name="litellm_core", kind="lib") -_GATEWAY_TARGET: Final = RustTarget( - package="litellm-ai-gateway", - name="litellm_ai_gateway", - kind="lib", -) -_AZURE_OCR_TESTS: Final = "providers::azure_ai::ocr::transformation::tests" -_MISTRAL_OCR_TESTS: Final = "providers::mistral::ocr::transformation::tests" -_VERTEX_OCR_TESTS: Final = "providers::vertex_ai::ocr::transformation::tests" -_REDUCTO_OCR_TESTS: Final = "providers::reducto::ocr::tests" -_GATEWAY_OCR_TESTS: Final = "ocr::tests" -_GATEWAY_PREPARE_OCR_TESTS: Final = "ocr::prepare::tests" - - -def _rust_test(target: RustTarget, module: str, test: str) -> RustTestIdentity: - return RustTestIdentity(target=target, name=f"{module}::{test}") - - -def _rust_family(target: RustTarget, module: str, test: str) -> RustTestFamily: - return RustTestFamily(target=target, name=f"{module}::{test}") - - -def _test_mappings(target: RustTarget, module: str, pairs: tuple[tuple[str, str], ...]) -> tuple[TestMapping, ...]: - return tuple(TestMapping(python=python, rust=_rust_test(target, module, test)) for python, test in pairs) - - -_AZURE_TRANSFORM_FILE: Final = "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py" -_AZURE_PAGES_FILE: Final = "tests/ocr_tests/test_ocr_azure_document_intelligence.py" -_AZURE_BASE_FILE: Final = "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py" -_RUST_BRIDGE_FILE: Final = "tests/test_litellm/ocr/test_rust_bridge.py" - -_AZURE_PORT_MAPPINGS: Final = _test_mappings( - _CORE_TARGET, - _AZURE_OCR_TESTS, - ( - ( - f"{_AZURE_TRANSFORM_FILE}::test_should_encode_azure_document_intelligence_model_id", - "azure_document_intelligence_model_id_is_encoded", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_should_reject_dot_segment_azure_document_intelligence_model_id", - "azure_document_intelligence_dot_segment_model_id_is_rejected", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_async_transform_ocr_response_preserves_azure_native_fields", - "document_intelligence_async_response_preserves_normalized_fields", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_transform_ocr_response_tolerates_missing_native_fields", - "document_intelligence_response_tolerates_missing_native_fields", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_transform_ocr_response_non_succeeded_status_raises", - "document_intelligence_non_succeeded_status_is_rejected", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_get_supported_ocr_params_includes_features", - "document_intelligence_supported_params_include_features", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_transform_ocr_response_native_format_carries_raw_operation", - "document_intelligence_native_format_carries_raw_operation", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_async_transform_ocr_response_native_format_carries_raw_operation", - "document_intelligence_async_native_format_carries_raw_operation", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_map_ocr_params_rejects_unknown_req_format_as_bad_request", - "document_intelligence_rejects_unknown_req_format", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_get_complete_url_omits_req_format_query_param", - "document_intelligence_url_omits_req_format", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_validate_environment_uses_subscription_key", - "document_intelligence_validate_environment_uses_subscription_key", - ), - ( - f"{_AZURE_TRANSFORM_FILE}::test_validate_environment_falls_back_to_entra_token", - "document_intelligence_validate_environment_falls_back_to_entra_token", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_get_supported_ocr_params_includes_pages_and_features", - "document_intelligence_supported_params_include_pages_features_and_req_format", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_mistral_zero_based_int_list", - "document_intelligence_maps_zero_based_page_list", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_dedupes_and_sorts", - "document_intelligence_page_mapping_dedupes_and_sorts", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_empty_list_omits_pages", - "document_intelligence_page_mapping_omits_empty_list", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_azure_native_string_range", - "document_intelligence_page_mapping_accepts_native_range", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_azure_native_string_with_spaces_stripped", - "document_intelligence_page_mapping_strips_spaces", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_list_of_string_tokens", - "document_intelligence_page_mapping_accepts_string_tokens", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_invalid_string_raises", - "document_intelligence_page_mapping_rejects_invalid_string", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_negative_index_raises", - "document_intelligence_page_mapping_rejects_negative_index", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_bool_list_raises", - "document_intelligence_page_mapping_rejects_bool_list", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_map_ocr_params_unsupported_type_raises", - "document_intelligence_page_mapping_rejects_unsupported_type", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_get_complete_url_appends_pages_query", - "document_intelligence_url_appends_pages_query", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_get_complete_url_no_pages_when_optional_params_empty", - "document_intelligence_url_has_no_pages_when_params_are_empty", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_transform_ocr_request_does_not_put_pages_in_body", - "document_intelligence_request_keeps_pages_out_of_body", - ), - ( - f"{_AZURE_PAGES_FILE}::TestAzureDocumentIntelligencePagesParam::test_end_to_end_mistral_shape_to_azure_query", - "document_intelligence_mistral_pages_flow_to_query_only", - ), - ( - "tests/test_litellm/llms/azure_ai/test_azure_ai_entra_auth.py::test_ocr_authenticates_with_entra_token", - "azure_ai_ocr_authenticates_with_entra_token", - ), - ( - f"{_AZURE_BASE_FILE}::TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_does_not_hijack_doc_intelligence", - "document_intelligence_endpoint_ignores_generic_azure_ai_base", - ), - ( - f"{_AZURE_BASE_FILE}::TestDocIntelligenceApiBaseResolution::test_explicit_api_base_is_honoured_for_doc_intelligence", - "document_intelligence_endpoint_honors_explicit_api_base", - ), - ( - f"{_AZURE_BASE_FILE}::TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_still_applies_to_mistral_ocr", - "azure_ai_mistral_ocr_uses_generic_api_base", - ), - ), -) - -_REDUCTO_PORT_MAPPINGS: Final = _test_mappings( - _CORE_TARGET, - _REDUCTO_OCR_TESTS, - ( - ( - "tests/test_litellm/llms/reducto/test_parse_v3.py::test_parse_v3_reducto_id_passthrough_skips_upload", - "test_parse_v3_reducto_id_passthrough_skips_upload", - ), - ( - "tests/test_litellm/llms/reducto/test_parse_legacy.py::test_parse_legacy_wraps_enhance_under_options", - "test_parse_legacy_wraps_enhance_under_options", - ), - ( - "tests/test_litellm/llms/reducto/test_upload.py::test_parse_v3_image_data_uri_upload_uses_image_mime", - "test_parse_v3_image_data_uri_upload_uses_image_mime", - ), - ( - "tests/test_litellm/llms/reducto/test_upload.py::test_parse_v3_uses_programmatic_api_key_over_env", - "test_parse_v3_uses_programmatic_api_key_over_env", - ), - ), -) - -_REDUCTO_GATEWAY_MAPPING: Final = TestMapping( - python="tests/test_litellm/llms/reducto/test_parse_v3.py::test_parse_v3_file_upload_and_response_mapping", - rust=_rust_test(_GATEWAY_TARGET, _GATEWAY_OCR_TESTS, "reducto_file_upload_then_parse_maps_response"), -) - -_GATEWAY_PORT_MAPPINGS: Final = _test_mappings( - _GATEWAY_TARGET, - _GATEWAY_PREPARE_OCR_TESTS, - ( - ( - "tests/test_litellm/ocr/test_ocr_native_format.py::test_native_format_rejected_for_provider_without_support_as_bad_request", - "native_format_rejected_for_provider_without_support_as_bad_request", - ), - ( - "tests/test_litellm/ocr/test_ocr_native_format.py::test_unknown_format_rejected_for_provider_without_support_as_bad_request", - "unknown_format_rejected_for_provider_without_support_as_bad_request", - ), - ), -) - -_HOST_ONLY_BRIDGE_EXCLUSIONS: Final = tuple( - MappingExclusionSpec(nodeid=f"{_RUST_BRIDGE_FILE}::{test}", reason=reason) - for test, reason in ( - ("test_ocr_routes_to_rust_when_enabled", "Python selects and invokes the native bridge."), - ("test_ocr_routes_azure_ai_to_rust_when_enabled", "Python resolves provider arguments before the bridge."), - ("test_ocr_rust_path_converts_file_document_before_bridge", "Python converts file inputs before the bridge."), - ( - "test_ocr_exception_type_uses_resolved_provider_context", - "Python wraps bridge exceptions into public errors.", - ), - ( - "test_rust_upstream_error_uses_ocr_provider_error_mapping", - "Python maps native upstream errors through the selected OCR provider config.", - ), - ("test_aocr_routes_to_async_rust_when_enabled", "Python selects and invokes the async native bridge."), - ("test_aocr_exception_type_uses_resolved_provider_context", "Python wraps async bridge exceptions."), - ("test_ocr_forwards_timeout_to_rust", "Python converts and forwards explicit timeouts."), - ("test_ocr_passes_default_request_timeout_to_rust", "Python supplies its process-level default timeout."), - ("test_ocr_falls_back_to_python_when_bridge_unavailable", "Python owns fallback when the extension is absent."), - ) -) - -_FAMILY_PORT_MAPPINGS: Final = ( - TestMapping( - python=f"{_AZURE_TRANSFORM_FILE}::test_transform_ocr_response_default_format_omits_raw_operation", - rust=_rust_family( - _CORE_TARGET, - _AZURE_OCR_TESTS, - "document_intelligence_default_format_omits_raw_operation", - ), - ), - TestMapping( - python=f"{_AZURE_TRANSFORM_FILE}::test_map_ocr_params_passes_through_req_format", - rust=_rust_family(_CORE_TARGET, _AZURE_OCR_TESTS, "document_intelligence_maps_req_format"), - ), - TestMapping( - python="tests/ocr_tests/test_ocr_vertex_ai.py::test_deepseek_request_uses_single_provider_namespace", - rust=_rust_family( - _CORE_TARGET, - _VERTEX_OCR_TESTS, - "vertex_deepseek_request_uses_single_provider_namespace", - ), - ), - TestMapping( - python="tests/test_litellm/llms/reducto/test_upload.py::test_parse_v3_rejects_plain_http_urls", - rust=_rust_family(_CORE_TARGET, _REDUCTO_OCR_TESTS, "test_parse_v3_rejects_plain_http_urls"), - ), -) - - -OCR_CONTRACT: Final = UnitTestContract( - mapping=MappingSpec( - python_functions=PythonFunctionDiscoverySpec( - trace_module="tests.rust-python-harness.strategies.trace_parity.sdk.ocr.case", - trace_spans=( - "ocr", - "prepare_ocr_call", - "ocr_provider_config", - "supported_ocr_params", - "map_ocr_params", - "validate_environment", - "complete_url", - "transform_ocr_request", - "execute_ocr_provider_call", - "transform_ocr_response", - "poll_document_intelligence", - ), - search_roots=("tests",), - exclude_roots=( - "tests/e2e", - "tests/ocr_tests/test_ocr_mistral.py", - "tests/rust-python-harness", - ), - includes=( - "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "tests/test_litellm/llms/mistral/ocr", - "tests/test_litellm/llms/ocr", - "tests/test_litellm/ocr", - "tests/test_litellm/proxy/ocr_endpoints", - ), - exclusions=( - "tests/ocr_tests/test_ocr_azure_document_intelligence.py::TestAzureDocumentIntelligenceOCR", - "tests/ocr_tests/test_ocr_vertex_ai.py::TestVertexAIMistralOCR", - "tests/ocr_tests/test_ocr_vertex_ai.py::TestVertexAIDeepSeekOCR", - ), - ), - rust_targets=(_CORE_TARGET, _GATEWAY_TARGET), - mappings=( - TestMapping( - python="tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py::test_transform_ocr_response_preserves_azure_native_fields", - rust=_rust_test(_CORE_TARGET, _AZURE_OCR_TESTS, "document_intelligence_response_normalizes_pages"), - ), - TestMapping( - python="tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py::test_map_ocr_params_features", - rust=_rust_family(_CORE_TARGET, _AZURE_OCR_TESTS, "document_intelligence_maps_features"), - ), - TestMapping( - python="tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py::test_map_ocr_params_empty_features_list_omitted", - rust=_rust_test(_CORE_TARGET, _AZURE_OCR_TESTS, "document_intelligence_url_omits_empty_feature_list"), - ), - TestMapping( - python="tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py::test_map_ocr_params_invalid_features_raises", - rust=_rust_family( - _CORE_TARGET, - _AZURE_OCR_TESTS, - "document_intelligence_mapping_rejects_invalid_features", - ), - ), - TestMapping( - python="tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py::test_get_complete_url_appends_features_query", - rust=_rust_test(_CORE_TARGET, _AZURE_OCR_TESTS, "document_intelligence_url_normalizes_features"), - ), - TestMapping( - python="tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py::test_get_complete_url_combines_pages_and_features", - rust=_rust_test( - _CORE_TARGET, _AZURE_OCR_TESTS, "document_intelligence_url_combines_pages_and_feature_list" - ), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestGetSupportedOcrParams::test_extract_header_in_supported_params", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "extract_header_is_a_supported_ocr_param"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestGetSupportedOcrParams::test_extract_footer_in_supported_params", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "extract_footer_is_a_supported_ocr_param"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestGetSupportedOcrParams::test_existing_params_still_present", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "existing_ocr_params_remain_supported"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestMapOcrParams::test_extract_header_passed_through", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "map_ocr_params_forwards_extract_header"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestMapOcrParams::test_extract_footer_passed_through", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "map_ocr_params_forwards_extract_footer"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestMapOcrParams::test_extract_header_and_footer_together", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "map_ocr_params_forwards_extract_header_and_footer"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestMapOcrParams::test_unknown_param_is_dropped", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "map_ocr_params_drops_unknown_params"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestNewSupportedParams::test_new_param_in_supported_list", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "new_ocr_params_are_supported"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestNewParamsMapOcr::test_new_param_passed_through", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "map_ocr_params_forwards_new_ocr_params"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestTransformOcrRequest::test_param_included_in_request_body", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "transform_ocr_request_includes_each_optional_param"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestTransformOcrRequest::test_multiple_new_params_together", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "transform_ocr_request_includes_multiple_new_params"), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestTransformOcrResponseOcr4Fields::test_blocks_and_confidence_scores_preserved", - rust=_rust_test( - _CORE_TARGET, _MISTRAL_OCR_TESTS, "transform_ocr_response_preserves_blocks_and_confidence_scores" - ), - ), - TestMapping( - python="tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py::TestTransformOcrResponseOcr4Fields::test_ocr4_fields_survive_model_dump", - rust=_rust_test(_CORE_TARGET, _MISTRAL_OCR_TESTS, "transform_ocr_response_preserves_ocr4_page_fields"), - ), - *_AZURE_PORT_MAPPINGS, - *_REDUCTO_PORT_MAPPINGS, - _REDUCTO_GATEWAY_MAPPING, - *_GATEWAY_PORT_MAPPINGS, - *_FAMILY_PORT_MAPPINGS, - ), - exclusions=_HOST_ONLY_BRIDGE_EXCLUSIONS, - require_complete=True, - ), - unit_parity=UnitParitySpec( - python_selectors=( - "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "tests/test_litellm/llms/mistral/ocr", - "tests/test_litellm/llms/ocr", - "tests/test_litellm/ocr", - ), - exclusions=( - UnitParityExclusionSpec( - nodeid="tests/test_litellm/ocr/test_rust_bridge.py::test_rust_toggles_flag", - reason="This test asserts the process-level backend flag selected by the parity runner.", - ), - ), - ), - rust=RustUnitSpec( - cargo_manifest="litellm-rust/Cargo.toml", - cargo_filter="ocr", - ), -) diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/contracts.py b/tests/rust-python-harness/strategies/unit_tests_mapping/contracts.py deleted file mode 100644 index a8f309cc8f3..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/contracts.py +++ /dev/null @@ -1,220 +0,0 @@ -from __future__ import annotations - -from collections import Counter -from typing import Final, Literal - -from pydantic import BaseModel, ConfigDict, field_validator, model_validator -from typing_extensions import Self - -from ...shared.tracing.pytest_usage import PythonFunctionReference -from ...shared.unit_runners.rust_runner import RustTarget, RustTestIdentity, RustTestScope - - -class _ContractModel(BaseModel): - model_config = ConfigDict(frozen=True, extra="forbid") - - -def _clean_unique(values: tuple[str, ...], field: str) -> tuple[str, ...]: - cleaned: Final = tuple(value.strip().rstrip("/") for value in values) - if not cleaned or any(not value for value in cleaned): - raise ValueError(f"{field} must contain non-empty paths") - duplicates: Final = tuple(value for value, count in Counter(cleaned).items() if count > 1) - if duplicates: - raise ValueError(f"{field} contains duplicates: {sorted(duplicates)}") - return cleaned - - -def _selector_contains(parent: str, child: str) -> bool: - return child == parent or child.startswith(f"{parent}/") - - -class RustTestFamily(_ContractModel): - kind: Literal["family"] = "family" - target: RustTarget - name: str - - @field_validator("name") - @classmethod - def validate_name(cls, value: str) -> str: - stripped: Final = value.strip() - if not stripped or stripped.endswith("::"): - raise ValueError("must be a non-empty Rust test base name") - return stripped - - @property - def key(self) -> str: - return f"{self.target.key}::{self.name}::case_*" - - def contains(self, identity: RustTestIdentity) -> bool: - return identity.target == self.target and identity.name.startswith(f"{self.name}::case_") - - -class TestMapping(_ContractModel): - python: str - rust: RustTestIdentity | RustTestFamily - - @field_validator("python") - @classmethod - def validate_python_nodeid(cls, value: str) -> str: - stripped: Final = value.strip() - if "::" not in stripped: - raise ValueError("must be a source path and test name separated by '::'") - return stripped - - -class PythonFunctionDiscoverySpec(_ContractModel): - functions: tuple[PythonFunctionReference, ...] = () - trace_module: str | None = None - trace_spans: tuple[str, ...] = () - search_roots: tuple[str, ...] - exclude_roots: tuple[str, ...] = () - includes: tuple[str, ...] = () - exclusions: tuple[str, ...] = () - - @field_validator("search_roots") - @classmethod - def validate_search_roots(cls, value: tuple[str, ...]) -> tuple[str, ...]: - return _clean_unique(value, "python function search_roots") - - @field_validator("exclude_roots") - @classmethod - def validate_exclude_roots(cls, value: tuple[str, ...]) -> tuple[str, ...]: - if not value: - return () - return _clean_unique(value, "python function exclude_roots") - - @model_validator(mode="after") - def validate_functions(self) -> Self: - if bool(self.functions) == bool(self.trace_module): - raise ValueError("python function discovery needs exactly one function list or trace module") - if self.trace_module is not None and not self.trace_spans: - raise ValueError("trace-derived Python function discovery needs trace_spans") - if not self.functions: - return self - keys: Final = tuple(f"{function.module}:{function.qualname}" for function in self.functions) - duplicates: Final = tuple(key for key, count in Counter(keys).items() if count > 1) - if duplicates: - raise ValueError(f"python function discovery contains duplicates: {sorted(duplicates)}") - return self - - -class UnitParityExclusionSpec(_ContractModel): - nodeid: str - reason: str - - @field_validator("nodeid", "reason") - @classmethod - def validate_fields(cls, value: str) -> str: - stripped: Final = value.strip() - if not stripped: - raise ValueError("must be a non-empty string") - return stripped - - -class MappingExclusionSpec(_ContractModel): - nodeid: str - reason: str - - @field_validator("nodeid", "reason") - @classmethod - def validate_fields(cls, value: str) -> str: - stripped: Final = value.strip() - if not stripped: - raise ValueError("must be a non-empty string") - return stripped - - -class MappingSpec(_ContractModel): - python_selectors: tuple[str, ...] = () - python_functions: PythonFunctionDiscoverySpec | None = None - rust_scope: tuple[RustTestScope, ...] = () - rust_targets: tuple[RustTarget, ...] = () - mappings: tuple[TestMapping, ...] - exclusions: tuple[MappingExclusionSpec, ...] = () - require_complete: bool = False - - @field_validator("python_selectors") - @classmethod - def validate_python_selectors(cls, value: tuple[str, ...]) -> tuple[str, ...]: - if not value: - return () - return _clean_unique(value, "python_selectors") - - @model_validator(mode="after") - def validate_rust_scope(self) -> Self: - if bool(self.python_selectors) == bool(self.python_functions): - raise ValueError("mapping needs exactly one Python selector or function-discovery scope") - targets: Final = tuple(scope.target.key for scope in self.rust_scope) - duplicates: Final = tuple(target for target, count in Counter(targets).items() if count > 1) - if duplicates: - raise ValueError(f"rust_scope contains duplicate targets: {sorted(duplicates)}") - target_names: Final = tuple(target.name for target in self.rust_targets) - duplicate_names: Final = tuple(name for name, count in Counter(target_names).items() if count > 1) - if duplicate_names: - raise ValueError(f"rust_targets contains duplicate names: {sorted(duplicate_names)}") - exclusion_nodeids: Final = tuple(exclusion.nodeid for exclusion in self.exclusions) - duplicate_exclusions: Final = tuple(nodeid for nodeid, count in Counter(exclusion_nodeids).items() if count > 1) - if duplicate_exclusions: - raise ValueError(f"mapping exclusions contain duplicate nodeids: {sorted(duplicate_exclusions)}") - return self - - -class UnitParitySpec(_ContractModel): - python_selectors: tuple[str, ...] - exclusions: tuple[UnitParityExclusionSpec, ...] = () - - @field_validator("python_selectors") - @classmethod - def validate_python_selectors(cls, value: tuple[str, ...]) -> tuple[str, ...]: - return _clean_unique(value, "unit parity python_selectors") - - @model_validator(mode="after") - def validate_exclusions(self) -> Self: - nodeids: Final = tuple(exclusion.nodeid for exclusion in self.exclusions) - duplicates: Final = tuple(nodeid for nodeid, count in Counter(nodeids).items() if count > 1) - if duplicates: - raise ValueError(f"unit parity exclusions contain duplicate nodeids: {sorted(duplicates)}") - return self - - -class RustUnitSpec(_ContractModel): - cargo_manifest: str - cargo_filter: str - cargo_package: str | None = None - - @field_validator("cargo_manifest", "cargo_filter") - @classmethod - def validate_required_fields(cls, value: str) -> str: - stripped: Final = value.strip() - if not stripped: - raise ValueError("must be a non-empty string") - return stripped - - @field_validator("cargo_package") - @classmethod - def validate_package(cls, value: str | None) -> str | None: - if value is None: - return None - stripped: Final = value.strip() - if not stripped: - raise ValueError("must be a non-empty string when provided") - return stripped - - -class UnitTestContract(_ContractModel): - mapping: MappingSpec - unit_parity: UnitParitySpec - rust: RustUnitSpec - - @model_validator(mode="after") - def validate_unit_parity_scope(self) -> Self: - if not self.mapping.python_selectors: - return self - unknown: Final = tuple( - selector - for selector in self.unit_parity.python_selectors - if not any(_selector_contains(parent, selector) for parent in self.mapping.python_selectors) - ) - if unknown: - raise ValueError(f"unit parity selectors must be contained in mapping selectors: {sorted(unknown)}") - return self diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/mapping_report.py b/tests/rust-python-harness/strategies/unit_tests_mapping/mapping_report.py deleted file mode 100644 index a5fd92e449d..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/mapping_report.py +++ /dev/null @@ -1,109 +0,0 @@ -from __future__ import annotations - -from collections import Counter -from collections.abc import Callable, Sequence -from typing import Final - -from pydantic import BaseModel, ConfigDict - -from .mapping_validator import MappingReport - - -class MappingReportArtifact(BaseModel): - model_config = ConfigDict(frozen=True, extra="forbid") - - report: MappingReport - detailed: bool = False - - -def _group_counts(nodeids: Sequence[str], owner: Callable[[str], str]) -> tuple[str, ...]: - counts: Final = Counter(owner(nodeid) for nodeid in nodeids) - width: Final = max((len(str(count)) for count in counts.values()), default=1) - return tuple( - f" {count:>{width}} {name}" for name, count in sorted(counts.items(), key=lambda item: (-item[1], item[0])) - ) - - -def _python_file(nodeid: str) -> str: - return nodeid.partition("::")[0] - - -def _rust_module(nodeid: str) -> str: - return nodeid.rpartition("::")[0] - - -def _details(nodeids: Sequence[str], owner: Callable[[str], str]) -> tuple[str, ...]: - owners: Final = tuple(sorted(frozenset(owner(nodeid) for nodeid in nodeids))) - return tuple( - line - for name in owners - for line in ( - f" {name}", - *(f" {nodeid.removeprefix(f'{name}::')}" for nodeid in nodeids if owner(nodeid) == name), - ) - ) - - -def _contract_errors(report: MappingReport) -> tuple[str, ...]: - return ( - *(f" Missing Python test: {nodeid}" for nodeid in report.missing_python_tests), - *(f" Missing Rust test: {nodeid}" for nodeid in report.missing_rust_tests), - *(f" Python test mapped more than once: {nodeid}" for nodeid in report.duplicate_python_mappings), - *(f" Rust test mapped more than once: {nodeid}" for nodeid in report.duplicate_rust_mappings), - *(f" Missing mapping exclusion: {nodeid}" for nodeid in report.invalid_mapping_exclusions), - *(f" Python test is both mapped and excluded: {nodeid}" for nodeid in report.mapped_and_excluded_python_tests), - *(f" Missing unit-parity exclusion: {nodeid}" for nodeid in report.invalid_unit_parity_exclusions), - ) - - -def mapping_report_lines(report: MappingReport, *, detailed: bool = False) -> tuple[str, ...]: - unmapped_count: Final = len(report.unmapped_python_tests) - excluded_count: Final = len(report.excluded_python_tests) - excluded_percentage: Final = ( - 0.0 if not report.total_count else round(100.0 * excluded_count / report.total_count, 1) - ) - unmapped_percentage: Final = ( - 0.0 if not report.total_count else round(100.0 * unmapped_count / report.total_count, 1) - ) - rust_total: Final = len(report.rust_tests) - rust_only_count: Final = len(report.rust_only_tests) - rust_mapped_count: Final = rust_total - rust_only_count - contract_errors: Final = _contract_errors(report) - detail_lines: Final = ( - ( - "", - "Unmapped Python test details", - *_details(report.unmapped_python_tests, _python_file), - "", - "Excluded Python test details", - *_details(report.excluded_python_tests, _python_file), - "", - "Rust-only test details", - *_details(report.rust_only_tests, _rust_module), - ) - if detailed - else () - ) - return ( - f"Contract: {'PASS' if report.is_valid else 'FAIL'}", - "", - "Python coverage", - f" Mapped {report.mapped_count:>3} / {report.total_count} ({report.percentage}%)", - f" Excluded {excluded_count:>3} / {report.total_count} ({excluded_percentage}%)", - f" Unmapped {unmapped_count:>3} / {report.total_count} ({unmapped_percentage}%)", - "", - "Rust inventory", - f" Mapped {rust_mapped_count:>3} / {rust_total}", - f" Rust-only {rust_only_count:>3} / {rust_total}", - "", - f"Unmapped Python tests by file ({unmapped_count})", - *_group_counts(report.unmapped_python_tests, _python_file), - "", - f"Excluded Python tests by file ({excluded_count})", - *_group_counts(report.excluded_python_tests, _python_file), - "", - f"Rust-only tests by module ({rust_only_count})", - *_group_counts(report.rust_only_tests, _rust_module), - *(("", "Contract errors", *contract_errors) if contract_errors else ()), - *detail_lines, - ) diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/mapping_validator.py b/tests/rust-python-harness/strategies/unit_tests_mapping/mapping_validator.py deleted file mode 100644 index 9dd79e860e6..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/mapping_validator.py +++ /dev/null @@ -1,296 +0,0 @@ -from __future__ import annotations - -import importlib -from collections import Counter, defaultdict -from collections.abc import Callable, Sequence -from pathlib import Path -from typing import Final, TypeAlias - -from pydantic import BaseModel, ConfigDict - -from ...shared.tracing.pytest_usage import ( - PythonFunctionIdentity, - RustFunctionIdentity, - candidate_test_files, - collect_python_function_tests, -) -from ...shared.tracing.steps import pipeline_projection -from ...shared.unit_runners.python_runner import collect_python_tests, contract_nodeid -from ...shared.unit_runners.rust_runner import RustTarget, RustTestIdentity, RustTestScope, enumerate_rust_tests -from .contracts import PythonFunctionDiscoverySpec, RustTestFamily, TestMapping, UnitTestContract - -PythonInventory: TypeAlias = Callable[[Sequence[str], Path], frozenset[str]] -RustInventory: TypeAlias = Callable[[Path, tuple[RustTestScope, ...]], frozenset[RustTestIdentity]] - - -def _trace_functions( - spec: PythonFunctionDiscoverySpec, -) -> tuple[tuple[PythonFunctionIdentity, ...], tuple[RustFunctionIdentity, ...]]: - from ..trace_parity.models import RouteSpec, TraceExecutionFailure, TraceSuite - from ..trace_parity.sdk.execution import collect_trace - - if spec.trace_module is None: - return () - module: Final = importlib.import_module(spec.trace_module) - suite: Final = getattr(module, "TRACE_SUITE", None) - if not isinstance(suite, TraceSuite) or not isinstance(suite.route, RouteSpec): - raise ValueError(f"{spec.trace_module} must export an SDK TRACE_SUITE") - python_functions: Final[dict[str, PythonFunctionIdentity]] = {} - rust_functions: Final[dict[str, RustFunctionIdentity]] = {} - for scenario in suite.scenarios: - route: Final = RouteSpec( - route=suite.route.route, - python_entrypoints=suite.route.python_entrypoints, - rust_entrypoints=suite.route.rust_entrypoints, - fixture=scenario.fixture, - ) - python_trace: Final = collect_trace(route, "python", asynchronous=scenario.asynchronous) - rust_trace: Final = collect_trace(route, "rust", asynchronous=scenario.asynchronous) - if isinstance(python_trace, TraceExecutionFailure): - raise ValueError(f"Python trace discovery failed for {scenario.name}: {python_trace.message}") - if isinstance(rust_trace, TraceExecutionFailure): - raise ValueError(f"Rust trace discovery failed for {scenario.name}: {rust_trace.message}") - python_projection: Final = pipeline_projection("python", python_trace, scenario.mappings) - rust_projection: Final = pipeline_projection("rust", rust_trace, scenario.mappings) - for step in python_projection.steps: - if step.span in spec.trace_spans: - function: Final = PythonFunctionIdentity.from_trace(step.raw) - python_functions[function.raw] = function - for step in rust_projection.steps: - if step.span in spec.trace_spans: - function: Final = RustFunctionIdentity.from_trace(step.raw) - rust_functions[step.raw] = function - if not python_functions or not rust_functions: - raise ValueError(f"Python trace discovery found no functions for spans: {', '.join(spec.trace_spans)}") - return ( - tuple(python_functions[key] for key in sorted(python_functions)), - tuple(rust_functions[key] for key in sorted(rust_functions)), - ) - - -def collect_python_function_inventory( - spec: PythonFunctionDiscoverySpec, - repo_root: Path, - traced_functions: Sequence[PythonFunctionIdentity] = (), -) -> frozenset[str]: - source_root: Final = repo_root / "litellm" - functions: Final = ( - tuple(reference.resolve(source_root) for reference in spec.functions) - if spec.functions - else tuple(traced_functions) - ) - discovered: Final = candidate_test_files( - functions, - spec.search_roots, - repo_root, - exclude_roots=spec.exclude_roots, - ) - selectors: Final = tuple(dict.fromkeys((*discovered, *spec.includes))) - if not selectors: - raise ValueError("Python function discovery found no candidate test files") - report: Final = collect_python_function_tests( - functions, - selectors, - repo_root, - source_root=source_root, - exclusions=spec.exclusions, - ) - if report.exit_code or report.problems: - details: Final = "\n".join(report.problems) or f"pytest exited with code {report.exit_code}" - raise ValueError(f"Python function test discovery failed:\n{details}") - return frozenset(contract_nodeid(nodeid) for usage in report.usages for nodeid in usage.tests) - - -def _colocated_rust_scope(mappings: Sequence[TestMapping]) -> tuple[RustTestScope, ...]: - modules_by_target: Final[dict[str, set[str]]] = defaultdict(set) - targets: Final[dict[str, RustTarget]] = {} - for item in mappings: - module, separator, _ = item.rust.name.partition("::tests::") - if not separator: - raise ValueError(f"Rust unit test is not colocated in a tests module: {item.rust.key}") - target_key: Final = item.rust.target.key - targets[target_key] = item.rust.target - modules_by_target[target_key].add(f"{module}::tests") - return tuple( - RustTestScope( - target=targets[target_key], - modules=tuple(sorted(modules_by_target[target_key])), - ) - for target_key in sorted(targets) - ) - - -def _traced_rust_scope( - functions: Sequence[RustFunctionIdentity], - targets: Sequence[RustTarget], - repo_root: Path, -) -> tuple[RustTestScope, ...]: - targets_by_name: Final = {target.name: target for target in targets} - modules_by_target: Final[dict[str, set[str]]] = defaultdict(set) - for function in functions: - crate: Final = function.module_path.partition("::")[0] - target: Final = targets_by_name.get(crate) - if target is None: - continue - source_candidates: Final = ( - repo_root / "litellm-rust" / function.file, - repo_root / function.file, - ) - source: Final = next((path for path in source_candidates if path.is_file()), None) - if source is None: - raise ValueError(f"Traced Rust source does not exist: {function.file}") - contents: Final = source.read_text() - if "mod tests" in contents and "#[cfg(test)]" in contents: - modules_by_target[target.key].add(function.test_module) - selected_targets: Final = {target.key: target for target in targets} - scopes: Final = tuple( - RustTestScope(target=selected_targets[key], modules=tuple(sorted(modules))) - for key, modules in sorted(modules_by_target.items()) - if modules - ) - if not scopes: - raise ValueError("Traced Rust functions have no colocated test modules") - return scopes - - -def _merge_rust_scopes(scopes: Sequence[RustTestScope]) -> tuple[RustTestScope, ...]: - targets: Final = {scope.target.key: scope.target for scope in scopes} - modules: Final[dict[str, set[str]]] = defaultdict(set) - features: Final[dict[str, set[str]]] = defaultdict(set) - default_features: Final[dict[str, bool]] = {} - for scope in scopes: - modules[scope.target.key].update(scope.modules) - features[scope.target.key].update(scope.features) - default_features[scope.target.key] = default_features.get(scope.target.key, True) and scope.default_features - return tuple( - RustTestScope( - target=targets[key], - modules=tuple( - sorted( - module - for module in modules[key] - if not any(module.startswith(f"{parent}::") for parent in modules[key]) - ) - ), - features=tuple(sorted(features[key])), - default_features=default_features[key], - ) - for key in sorted(targets) - ) - - -def _owned_rust_tests( - rust: RustTestIdentity | RustTestFamily, - inventory: frozenset[RustTestIdentity], -) -> frozenset[RustTestIdentity]: - if isinstance(rust, RustTestFamily): - return frozenset(identity for identity in inventory if rust.contains(identity)) - return frozenset((rust,)) if rust in inventory else frozenset() - - -class MappingReport(BaseModel): - model_config = ConfigDict(frozen=True, extra="forbid") - - python_tests: tuple[str, ...] - rust_tests: tuple[str, ...] - mapped_python_tests: tuple[str, ...] - excluded_python_tests: tuple[str, ...] - unmapped_python_tests: tuple[str, ...] - rust_only_tests: tuple[str, ...] - missing_python_tests: tuple[str, ...] - missing_rust_tests: tuple[str, ...] - duplicate_python_mappings: tuple[str, ...] - duplicate_rust_mappings: tuple[str, ...] - invalid_mapping_exclusions: tuple[str, ...] - mapped_and_excluded_python_tests: tuple[str, ...] - invalid_unit_parity_exclusions: tuple[str, ...] - - @property - def mapped_count(self) -> int: - return len(self.mapped_python_tests) - - @property - def total_count(self) -> int: - return len(self.python_tests) - - @property - def percentage(self) -> float: - return 0.0 if not self.total_count else round(100.0 * self.mapped_count / self.total_count, 1) - - @property - def is_valid(self) -> bool: - return not ( - self.missing_python_tests - or self.missing_rust_tests - or self.duplicate_python_mappings - or self.duplicate_rust_mappings - or self.invalid_mapping_exclusions - or self.mapped_and_excluded_python_tests - or self.invalid_unit_parity_exclusions - ) - - -def audit_mapping( - contract: UnitTestContract, - repo_root: Path, - *, - python_inventory: PythonInventory = collect_python_tests, - rust_inventory: RustInventory = enumerate_rust_tests, -) -> MappingReport: - mapping: Final = contract.mapping - traced_python: tuple[PythonFunctionIdentity, ...] = () - traced_rust: tuple[RustFunctionIdentity, ...] = () - if mapping.python_functions is not None and mapping.python_functions.trace_module is not None: - traced_python, traced_rust = _trace_functions(mapping.python_functions) - python_tests: Final = ( - collect_python_function_inventory(mapping.python_functions, repo_root, traced_python) - if mapping.python_functions is not None - else python_inventory(mapping.python_selectors, repo_root) - ) - unit_parity_tests: Final = python_inventory(contract.unit_parity.python_selectors, repo_root) - traced_scope: Final = _traced_rust_scope(traced_rust, mapping.rust_targets, repo_root) if traced_rust else () - rust_scope: Final = _merge_rust_scopes( - (*mapping.rust_scope, *traced_scope, *_colocated_rust_scope(mapping.mappings)) - ) - rust_tests: Final = rust_inventory(repo_root, rust_scope) - mapped_python: Final = frozenset(item.python for item in mapping.mappings) - excluded_python: Final = frozenset(exclusion.nodeid for exclusion in mapping.exclusions) - rust_ownership: Final = tuple((item.rust, _owned_rust_tests(item.rust, rust_tests)) for item in mapping.mappings) - mapped_rust: Final = frozenset(identity for _, identities in rust_ownership for identity in identities) - duplicate_python: Final = tuple( - sorted(nodeid for nodeid, count in Counter(item.python for item in mapping.mappings).items() if count > 1) - ) - duplicate_exact_rust: Final = frozenset( - identity.key - for identity, count in Counter( - item.rust for item in mapping.mappings if isinstance(item.rust, RustTestIdentity) - ).items() - if count > 1 - ) - duplicate_owned_rust: Final = frozenset( - identity.key - for identity, count in Counter(identity for _, identities in rust_ownership for identity in identities).items() - if count > 1 - ) - duplicate_rust: Final = tuple(sorted(duplicate_exact_rust | duplicate_owned_rust)) - return MappingReport( - python_tests=tuple(sorted(python_tests)), - rust_tests=tuple(sorted(identity.key for identity in rust_tests)), - mapped_python_tests=tuple(sorted(python_tests & mapped_python)), - excluded_python_tests=tuple(sorted((python_tests & excluded_python) - mapped_python)), - unmapped_python_tests=tuple(sorted(python_tests - mapped_python - excluded_python)), - rust_only_tests=tuple(sorted(identity.key for identity in rust_tests - mapped_rust)), - missing_python_tests=tuple(sorted(mapped_python - python_tests)), - missing_rust_tests=tuple(sorted(rust.key for rust, identities in rust_ownership if not identities)), - duplicate_python_mappings=duplicate_python, - duplicate_rust_mappings=duplicate_rust, - invalid_mapping_exclusions=tuple(sorted(excluded_python - python_tests)), - mapped_and_excluded_python_tests=tuple(sorted(mapped_python & excluded_python)), - invalid_unit_parity_exclusions=tuple( - sorted( - exclusion.nodeid - for exclusion in contract.unit_parity.exclusions - if exclusion.nodeid not in unit_parity_tests - ) - ), - ) diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/mappings.py b/tests/rust-python-harness/strategies/unit_tests_mapping/mappings.py deleted file mode 100644 index efb5b2a644a..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/mappings.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import annotations - -from collections.abc import Mapping -from types import MappingProxyType -from typing import Final - -from ...shared.reporting.models import SdkFunction -from .cases.ocr import OCR_CONTRACT -from .contracts import UnitTestContract - -UNIT_TEST_CONTRACTS: Final[Mapping[SdkFunction, UnitTestContract]] = MappingProxyType({"ocr": OCR_CONTRACT}) diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/reporting.py b/tests/rust-python-harness/strategies/unit_tests_mapping/reporting.py deleted file mode 100644 index d4bce7bc768..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/reporting.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence -from typing import Final - -from pydantic import ValidationError - -from ...shared.reporting.models import CaseResult -from ...shared.reporting.rendering import ReportSection, render_case_outcome -from .mapping_report import MappingReportArtifact, mapping_report_lines -from .runner import MAPPING_REPORT_ARTIFACT - - -def _render_artifact(body: str) -> str: - try: - artifact: Final = MappingReportArtifact.model_validate_json(body) - except ValidationError as error: - return f"Mapping report artifact is invalid: {error}" - return "\n".join(mapping_report_lines(artifact.report, detailed=artifact.detailed)) - - -def _render_result(result: CaseResult) -> str: - reports: Final = tuple( - _render_artifact(artifact.body) - for artifacts in result.artifacts.values() - for artifact in artifacts - if artifact.kind == MAPPING_REPORT_ARTIFACT - ) - if reports: - return "\n".join((f"Case: {result.case.display_name}", *reports)) - return render_case_outcome(result) - - -def render_mapping_results(results: Sequence[CaseResult]) -> tuple[ReportSection, ...]: - blocks: Final = tuple(_render_result(result) for result in results) - return (ReportSection("Python/Rust unit-test mappings", blocks or ("No mapping cases selected",)),) diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/runner.py b/tests/rust-python-harness/strategies/unit_tests_mapping/runner.py deleted file mode 100644 index 540edca9385..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/runner.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence -from pathlib import Path -from typing import Final - -from ...shared.native_build import ensure_trace_bridge -from ...shared.reporting.models import ResultArtifact -from ...shared.unit_runners.python_runner import collect_python_tests -from ...shared.unit_runners.rust_runner import enumerate_rust_tests -from ...shared.unit_runners.suite_runner import SuiteExecution -from .contracts import UnitTestContract -from .mapping_report import MappingReportArtifact -from .mapping_validator import PythonInventory, RustInventory, audit_mapping - -MAPPING_REPORT_ARTIFACT: Final = "mapping_report" - - -def _audit_problems(artifact: MappingReportArtifact) -> tuple[str, ...]: - report: Final = artifact.report - return ( - *(f"mapped Python test does not exist: {nodeid}" for nodeid in report.missing_python_tests), - *(f"mapped Rust test does not exist: {nodeid}" for nodeid in report.missing_rust_tests), - *(f"Python test has multiple mappings: {nodeid}" for nodeid in report.duplicate_python_mappings), - *(f"Rust test has multiple mappings: {nodeid}" for nodeid in report.duplicate_rust_mappings), - *(f"mapping exclusion does not exist: {nodeid}" for nodeid in report.invalid_mapping_exclusions), - *(f"Python test is both mapped and excluded: {nodeid}" for nodeid in report.mapped_and_excluded_python_tests), - *(f"unit parity exclusion does not exist: {nodeid}" for nodeid in report.invalid_unit_parity_exclusions), - ) - - -def run_suite( - contract: UnitTestContract, - repo_root: Path, - runner_args: Sequence[str] = (), - *, - python_inventory: PythonInventory = collect_python_tests, - rust_inventory: RustInventory = enumerate_rust_tests, -) -> SuiteExecution: - if contract.mapping.python_functions is not None and contract.mapping.python_functions.trace_module is not None: - bridge_error: Final = ensure_trace_bridge(repo_root) - if bridge_error is not None: - return SuiteExecution(problems=(bridge_error,)) - artifact: Final = MappingReportArtifact( - report=audit_mapping( - contract, - repo_root, - python_inventory=python_inventory, - rust_inventory=rust_inventory, - ), - detailed=bool(runner_args), - ) - completeness_problems: Final = ( - tuple(f"Python test has no Rust mapping: {nodeid}" for nodeid in artifact.report.unmapped_python_tests) - if contract.mapping.require_complete - else () - ) - return SuiteExecution( - problems=(*_audit_problems(artifact), *completeness_problems), - artifacts=(ResultArtifact(MAPPING_REPORT_ARTIFACT, artifact.model_dump_json()),), - ) diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/test_mapping_validator.py b/tests/rust-python-harness/strategies/unit_tests_mapping/test_mapping_validator.py deleted file mode 100644 index 6635a0eb522..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/test_mapping_validator.py +++ /dev/null @@ -1,314 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -from typing import Final - -import pytest -from pydantic import ValidationError - -from ...shared.unit_runners.rust_runner import RustTarget, RustTestIdentity, RustTestScope -from .contracts import ( - MappingExclusionSpec, - MappingSpec, - RustTestFamily, - RustUnitSpec, - UnitParityExclusionSpec, - UnitParitySpec, - UnitTestContract, -) -from .contracts import TestMapping as MappingPair -from .mapping_validator import audit_mapping - -_TARGET: Final = RustTarget(package="example", name="example", kind="lib") -_SCOPE: Final = RustTestScope(target=_TARGET, modules=("api::tests",)) -_PYTHON_TESTS: Final = frozenset(("test_api.py::test_decode", "test_api.py::test_unmapped")) -_RUST_TEST: Final = RustTestIdentity(target=_TARGET, name="api::tests::decodes") -_RUST_ONLY: Final = RustTestIdentity(target=_TARGET, name="api::tests::rust_only") -_RUST_TESTS: Final = frozenset((_RUST_TEST, _RUST_ONLY)) - - -def _python_inventory(*_: object) -> frozenset[str]: - return _PYTHON_TESTS - - -def _rust_inventory(*_: object) -> frozenset[RustTestIdentity]: - return _RUST_TESTS - - -def _contract(*mappings: MappingPair, exclusions: tuple[UnitParityExclusionSpec, ...] = ()) -> UnitTestContract: - return UnitTestContract( - mapping=MappingSpec( - python_selectors=("test_api.py",), - rust_scope=(_SCOPE,), - mappings=mappings, - ), - unit_parity=UnitParitySpec(python_selectors=("test_api.py",), exclusions=exclusions), - rust=RustUnitSpec(cargo_manifest="Cargo.toml", cargo_filter="api"), - ) - - -def _mapping_exclusion(nodeid: str) -> MappingExclusionSpec: - return MappingExclusionSpec(nodeid=nodeid, reason="Python bridge availability is host-only") - - -def test_derives_mapping_status_from_live_inventories(tmp_path: Path) -> None: - contract: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST)) - - report: Final = audit_mapping( - contract, tmp_path, python_inventory=_python_inventory, rust_inventory=_rust_inventory - ) - - assert report.is_valid - assert report.mapped_python_tests == ("test_api.py::test_decode",) - assert report.unmapped_python_tests == ("test_api.py::test_unmapped",) - assert report.rust_only_tests == (_RUST_ONLY.key,) - assert report.percentage == 50.0 - - -def test_reports_stale_and_duplicate_mappings(tmp_path: Path) -> None: - removed: Final = RustTestIdentity(target=_TARGET, name="api::tests::removed") - contract: Final = _contract( - MappingPair(python="test_api.py::removed", rust=removed), - MappingPair(python="test_api.py::removed", rust=_RUST_TEST), - ) - - report: Final = audit_mapping( - contract, tmp_path, python_inventory=_python_inventory, rust_inventory=_rust_inventory - ) - - assert not report.is_valid - assert report.missing_python_tests == ("test_api.py::removed",) - assert report.missing_rust_tests == (removed.key,) - assert report.duplicate_python_mappings == ("test_api.py::removed",) - - -def test_reports_duplicate_rust_mapping_and_invalid_exclusion(tmp_path: Path) -> None: - contract: Final = _contract( - MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST), - MappingPair(python="test_api.py::test_unmapped", rust=_RUST_TEST), - exclusions=(UnitParityExclusionSpec(nodeid="test_api.py::removed", reason="Removed test"),), - ) - - report: Final = audit_mapping( - contract, tmp_path, python_inventory=_python_inventory, rust_inventory=_rust_inventory - ) - - assert not report.is_valid - assert report.duplicate_rust_mappings == (_RUST_TEST.key,) - assert report.invalid_unit_parity_exclusions == ("test_api.py::removed",) - - -def test_excludes_host_only_python_test_from_unmapped_inventory(tmp_path: Path) -> None: - partial: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST)) - contract: Final = partial.model_copy( - update={ - "mapping": partial.mapping.model_copy( - update={"exclusions": (_mapping_exclusion("test_api.py::test_unmapped"),)} - ) - } - ) - - report: Final = audit_mapping( - contract, tmp_path, python_inventory=_python_inventory, rust_inventory=_rust_inventory - ) - - assert report.is_valid - assert report.excluded_python_tests == ("test_api.py::test_unmapped",) - assert report.unmapped_python_tests == () - - -def test_reports_missing_and_mapped_mapping_exclusions(tmp_path: Path) -> None: - partial: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST)) - contract: Final = partial.model_copy( - update={ - "mapping": partial.mapping.model_copy( - update={ - "exclusions": ( - _mapping_exclusion("test_api.py::test_decode"), - _mapping_exclusion("test_api.py::removed"), - ) - } - ) - } - ) - - report: Final = audit_mapping( - contract, tmp_path, python_inventory=_python_inventory, rust_inventory=_rust_inventory - ) - - assert not report.is_valid - assert report.invalid_mapping_exclusions == ("test_api.py::removed",) - assert report.mapped_and_excluded_python_tests == ("test_api.py::test_decode",) - - -def test_resolves_rstest_family_to_generated_cases(tmp_path: Path) -> None: - first_case: Final = RustTestIdentity(target=_TARGET, name="api::tests::decodes::case_1_png") - second_case: Final = RustTestIdentity(target=_TARGET, name="api::tests::decodes::case_2_pdf") - family: Final = RustTestFamily(target=_TARGET, name="api::tests::decodes") - contract: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=family)) - - report: Final = audit_mapping( - contract, - tmp_path, - python_inventory=_python_inventory, - rust_inventory=lambda *_: frozenset((first_case, second_case)), - ) - - assert report.is_valid - assert report.mapped_python_tests == ("test_api.py::test_decode",) - assert report.missing_rust_tests == () - - -def test_reports_missing_rstest_family(tmp_path: Path) -> None: - family: Final = RustTestFamily(target=_TARGET, name="api::tests::decodes") - contract: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=family)) - - report: Final = audit_mapping( - contract, tmp_path, python_inventory=_python_inventory, rust_inventory=_rust_inventory - ) - - assert not report.is_valid - assert report.missing_rust_tests == (family.key,) - - -def test_reports_concrete_test_owned_by_exact_and_family_mappings(tmp_path: Path) -> None: - generated: Final = RustTestIdentity(target=_TARGET, name="api::tests::decodes::case_1_png") - family: Final = RustTestFamily(target=_TARGET, name="api::tests::decodes") - contract: Final = _contract( - MappingPair(python="test_api.py::test_decode", rust=family), - MappingPair(python="test_api.py::test_unmapped", rust=generated), - ) - - report: Final = audit_mapping( - contract, - tmp_path, - python_inventory=_python_inventory, - rust_inventory=lambda *_: frozenset((generated,)), - ) - - assert not report.is_valid - assert report.duplicate_rust_mappings == (generated.key,) - - -def test_rstest_family_cases_are_not_rust_only(tmp_path: Path) -> None: - generated: Final = RustTestIdentity(target=_TARGET, name="api::tests::decodes::case_1_png") - unrelated: Final = RustTestIdentity(target=_TARGET, name="api::tests::rust_only") - family: Final = RustTestFamily(target=_TARGET, name="api::tests::decodes") - contract: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=family)) - - report: Final = audit_mapping( - contract, - tmp_path, - python_inventory=_python_inventory, - rust_inventory=lambda *_: frozenset((generated, unrelated)), - ) - - assert report.rust_only_tests == (unrelated.key,) - - -def test_merges_configured_and_colocated_rust_scopes(tmp_path: Path) -> None: - support_test: Final = RustTestIdentity(target=_TARGET, name="support::tests::rust_only") - configured_scope: Final = RustTestScope( - target=_TARGET, - modules=("support::tests",), - features=("mock",), - default_features=False, - ) - expected_scope: Final = RustTestScope( - target=_TARGET, - modules=("api::tests", "support::tests"), - features=("mock",), - default_features=False, - ) - contract: Final = UnitTestContract( - mapping=MappingSpec( - python_selectors=("test_api.py",), - rust_scope=(configured_scope,), - mappings=(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST),), - ), - unit_parity=UnitParitySpec(python_selectors=("test_api.py",)), - rust=RustUnitSpec(cargo_manifest="Cargo.toml", cargo_filter="api"), - ) - - def assert_merged_scope(_: Path, scopes: tuple[RustTestScope, ...]) -> frozenset[RustTestIdentity]: - assert scopes == (expected_scope,) - return frozenset((_RUST_TEST, support_test)) - - report: Final = audit_mapping( - contract, - tmp_path, - python_inventory=_python_inventory, - rust_inventory=assert_merged_scope, - ) - - assert report.is_valid - assert report.rust_only_tests == (support_test.key,) - - -def test_merged_rust_scope_removes_modules_contained_by_parent(tmp_path: Path) -> None: - expected_scope: Final = RustTestScope(target=_TARGET, modules=("api",)) - contract: Final = UnitTestContract( - mapping=MappingSpec( - python_selectors=("test_api.py",), - rust_scope=(expected_scope,), - mappings=(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST),), - ), - unit_parity=UnitParitySpec(python_selectors=("test_api.py",)), - rust=RustUnitSpec(cargo_manifest="Cargo.toml", cargo_filter="api"), - ) - - def assert_parent_scope(_: Path, scopes: tuple[RustTestScope, ...]) -> frozenset[RustTestIdentity]: - assert scopes == (expected_scope,) - return frozenset((_RUST_TEST,)) - - report: Final = audit_mapping( - contract, - tmp_path, - python_inventory=_python_inventory, - rust_inventory=assert_parent_scope, - ) - - assert report.is_valid - - -def test_accepts_descendant_unit_parity_selector() -> None: - contract: Final = UnitTestContract( - mapping=MappingSpec(python_selectors=("tests/api",), rust_scope=(_SCOPE,), mappings=()), - unit_parity=UnitParitySpec(python_selectors=("tests/api/test_ocr.py",)), - rust=RustUnitSpec(cargo_manifest="Cargo.toml", cargo_filter="api"), - ) - - assert contract.unit_parity.python_selectors == ("tests/api/test_ocr.py",) - - -@pytest.mark.parametrize( - "mapping_selectors,parity_selectors", - (((), ("tests/api",)), (("tests/api", "tests/api"), ("tests/api",)), (("tests/api",), ("tests/chat",))), -) -def test_rejects_invalid_selector_contracts( - mapping_selectors: tuple[str, ...], parity_selectors: tuple[str, ...] -) -> None: - with pytest.raises(ValidationError): - UnitTestContract( - mapping=MappingSpec(python_selectors=mapping_selectors, rust_scope=(_SCOPE,), mappings=()), - unit_parity=UnitParitySpec(python_selectors=parity_selectors), - rust=RustUnitSpec(cargo_manifest="Cargo.toml", cargo_filter="api"), - ) - - -def test_rejects_duplicate_scopes_and_exclusions() -> None: - exclusion: Final = UnitParityExclusionSpec(nodeid="test_api.py::test_skip", reason="Backend assertion") - with pytest.raises(ValidationError, match="duplicate targets"): - MappingSpec(python_selectors=("test_api.py",), rust_scope=(_SCOPE, _SCOPE), mappings=()) - with pytest.raises(ValidationError, match="duplicate nodeids"): - UnitParitySpec(python_selectors=("test_api.py",), exclusions=(exclusion, exclusion)) - mapping_exclusion: Final = _mapping_exclusion("test_api.py::test_skip") - with pytest.raises(ValidationError, match="mapping exclusions contain duplicate nodeids"): - MappingSpec( - python_selectors=("test_api.py",), - rust_scope=(_SCOPE,), - mappings=(), - exclusions=(mapping_exclusion, mapping_exclusion), - ) - with pytest.raises(ValidationError, match="must be a non-empty string"): - MappingExclusionSpec(nodeid="test_api.py::test_skip", reason=" ") diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/test_reporting.py b/tests/rust-python-harness/strategies/unit_tests_mapping/test_reporting.py deleted file mode 100644 index 36e18a9d109..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/test_reporting.py +++ /dev/null @@ -1,99 +0,0 @@ -from __future__ import annotations - -from typing import Final - -from ...shared.reporting.models import CaseResult, Coverage, HarnessCase, ResultArtifact, RunStatus -from ...shared.reporting.strategy import SuiteCaseSpec -from .mapping_report import MappingReportArtifact -from .mapping_validator import MappingReport -from .reporting import render_mapping_results -from .runner import MAPPING_REPORT_ARTIFACT - - -def _report(*, invalid: bool = False, excluded: bool = False) -> MappingReport: - return MappingReport( - python_tests=("test_api.py::test_decode", "test_api.py::test_unmapped"), - rust_tests=("example/lib/example::api::tests::decodes", "example/lib/example::api::tests::rust_only"), - mapped_python_tests=("test_api.py::test_decode",), - excluded_python_tests=(("test_api.py::test_unmapped",) if excluded else ()), - unmapped_python_tests=(() if excluded else ("test_api.py::test_unmapped",)), - rust_only_tests=("example/lib/example::api::tests::rust_only",), - missing_python_tests=("test_api.py::removed",) if invalid else (), - missing_rust_tests=(), - duplicate_python_mappings=(), - duplicate_rust_mappings=(), - invalid_mapping_exclusions=(), - mapped_and_excluded_python_tests=(), - invalid_unit_parity_exclusions=(), - ) - - -def _result(body: str) -> CaseResult: - case: Final = HarnessCase( - strategy_id="unit_tests_mapping", - strategy_label="Unit test mapping", - sdk_function="ocr", - spec=SuiteCaseSpec(coverage=Coverage.COMPLETE, suite="ocr"), - ) - result: Final = CaseResult(case=case) - result.record( - "suite:unit_tests_mapping:ocr:ocr", - RunStatus.PASSED, - artifacts=(ResultArtifact(MAPPING_REPORT_ARTIFACT, body),), - ) - return result - - -def test_renderer_preserves_summary_and_detailed_output() -> None: - summary: Final = MappingReportArtifact(report=_report()).model_dump_json() - detailed: Final = MappingReportArtifact(report=_report(), detailed=True).model_dump_json() - - summary_text: Final = "\n".join(render_mapping_results((_result(summary),))[0].blocks) - detailed_text: Final = "\n".join(render_mapping_results((_result(detailed),))[0].blocks) - - assert "Mapped 1 / 2 (50.0%)" in summary_text - assert "Unmapped Python test details" not in summary_text - assert "Unmapped Python test details\n test_api.py\n test_unmapped" in detailed_text - assert "Rust-only test details" in detailed_text - - -def test_renderer_shows_contract_errors() -> None: - body: Final = MappingReportArtifact(report=_report(invalid=True)).model_dump_json() - rendered: Final = "\n".join(render_mapping_results((_result(body),))[0].blocks) - - assert "Contract: FAIL" in rendered - assert "Missing Python test: test_api.py::removed" in rendered - - -def test_renderer_distinguishes_excluded_python_tests() -> None: - body: Final = MappingReportArtifact(report=_report(excluded=True), detailed=True).model_dump_json() - rendered: Final = "\n".join(render_mapping_results((_result(body),))[0].blocks) - - assert "Excluded 1 / 2 (50.0%)" in rendered - assert "Unmapped 0 / 2 (0.0%)" in rendered - assert "Excluded Python test details\n test_api.py\n test_unmapped" in rendered - - -def test_renderer_handles_empty_inventory_and_malformed_artifact() -> None: - empty: Final = MappingReport( - python_tests=(), - rust_tests=(), - mapped_python_tests=(), - excluded_python_tests=(), - unmapped_python_tests=(), - rust_only_tests=(), - missing_python_tests=(), - missing_rust_tests=(), - duplicate_python_mappings=(), - duplicate_rust_mappings=(), - invalid_mapping_exclusions=(), - mapped_and_excluded_python_tests=(), - invalid_unit_parity_exclusions=(), - ) - empty_text: Final = "\n".join( - render_mapping_results((_result(MappingReportArtifact(report=empty).model_dump_json()),))[0].blocks - ) - invalid_text: Final = "\n".join(render_mapping_results((_result("not-json"),))[0].blocks) - - assert "Mapped 0 / 0 (0.0%)" in empty_text - assert "Mapping report artifact is invalid:" in invalid_text diff --git a/tests/rust-python-harness/strategies/unit_tests_mapping/test_runner.py b/tests/rust-python-harness/strategies/unit_tests_mapping/test_runner.py deleted file mode 100644 index 2b14c716e1d..00000000000 --- a/tests/rust-python-harness/strategies/unit_tests_mapping/test_runner.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import annotations - -from functools import partial -from pathlib import Path -from typing import Final - -from ...shared.reporting.models import Coverage, HarnessCase, RunStatus -from ...shared.reporting.strategy import SuiteCaseSpec -from ...shared.unit_runners.rust_runner import RustTarget, RustTestIdentity, RustTestScope -from ...shared.unit_runners.suite_runner import run_suites -from .contracts import ( - MappingExclusionSpec, - MappingSpec, - RustUnitSpec, - TestMapping as MappingPair, - UnitParitySpec, - UnitTestContract, -) -from .mapping_report import MappingReportArtifact -from .runner import MAPPING_REPORT_ARTIFACT, run_suite - -_TARGET: Final = RustTarget(package="example", name="example", kind="lib") -_RUST_TEST: Final = RustTestIdentity(target=_TARGET, name="api::tests::decodes") -_RUST_ONLY: Final = RustTestIdentity(target=_TARGET, name="api::tests::rust_only") - - -def _python_inventory(*_: object) -> frozenset[str]: - return frozenset(("test_api.py::test_decode", "test_api.py::test_unmapped")) - - -def _rust_inventory(*_: object) -> frozenset[RustTestIdentity]: - return frozenset((_RUST_TEST, _RUST_ONLY)) - - -def _contract(mapping: MappingPair) -> UnitTestContract: - return UnitTestContract( - mapping=MappingSpec( - python_selectors=("test_api.py",), - rust_scope=(RustTestScope(target=_TARGET, modules=("api::tests",)),), - mappings=(mapping,), - ), - unit_parity=UnitParitySpec(python_selectors=("test_api.py",)), - rust=RustUnitSpec(cargo_manifest="Cargo.toml", cargo_filter="api"), - ) - - -def _case() -> HarnessCase: - return HarnessCase( - strategy_id="unit_tests_mapping", - strategy_label="Unit test mapping", - sdk_function="ocr", - spec=SuiteCaseSpec(coverage=Coverage.COMPLETE, suite="ocr"), - ) - - -def test_reports_structured_mapping_status_without_running_tests(tmp_path: Path) -> None: - contract: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST)) - case: Final = _case() - - code, report = run_suites( - (case,), - tmp_path, - lambda _: None, - suites={"ocr": contract}, - execute=partial( - run_suite, - python_inventory=_python_inventory, - rust_inventory=_rust_inventory, - ), - ) - - result: Final = report.results[case.key] - artifacts: Final = tuple( - artifact - for values in result.artifacts.values() - for artifact in values - if artifact.kind == MAPPING_REPORT_ARTIFACT - ) - parsed: Final = MappingReportArtifact.model_validate_json(artifacts[0].body) - assert code == 0, report.failures - assert result.status is RunStatus.PASSED - assert parsed.report.mapped_count == 1 - assert parsed.report.total_count == 2 - assert not parsed.detailed - - -def test_fails_when_a_mapping_target_is_missing(tmp_path: Path) -> None: - missing: Final = RustTestIdentity(target=_TARGET, name="api::tests::missing") - contract: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=missing)) - case: Final = _case() - - code, report = run_suites( - (case,), - tmp_path, - lambda _: None, - suites={"ocr": contract}, - execute=partial( - run_suite, - python_inventory=_python_inventory, - rust_inventory=_rust_inventory, - ), - ) - - assert code == 1 - assert report.results[case.key].status is RunStatus.FAILED - assert any("mapped Rust test does not exist" in detail for _, detail in report.failures) - - -def test_required_complete_mapping_fails_for_unmapped_python_test(tmp_path: Path) -> None: - partial: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST)) - contract: Final = partial.model_copy( - update={"mapping": partial.mapping.model_copy(update={"require_complete": True})} - ) - - execution: Final = run_suite( - contract, - tmp_path, - python_inventory=_python_inventory, - rust_inventory=_rust_inventory, - ) - - assert execution.problems == ("Python test has no Rust mapping: test_api.py::test_unmapped",) - - -def test_required_complete_mapping_accepts_host_only_exclusion(tmp_path: Path) -> None: - partial: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST)) - contract: Final = partial.model_copy( - update={ - "mapping": partial.mapping.model_copy( - update={ - "require_complete": True, - "exclusions": ( - MappingExclusionSpec( - nodeid="test_api.py::test_unmapped", - reason="Python bridge availability is host-only", - ), - ), - } - ) - } - ) - - execution: Final = run_suite( - contract, - tmp_path, - python_inventory=_python_inventory, - rust_inventory=_rust_inventory, - ) - artifact: Final = MappingReportArtifact.model_validate_json(execution.artifacts[0].body) - - assert execution.problems == () - assert artifact.report.excluded_python_tests == ("test_api.py::test_unmapped",) - - -def test_detail_argument_is_stored_in_artifact(tmp_path: Path) -> None: - contract: Final = _contract(MappingPair(python="test_api.py::test_decode", rust=_RUST_TEST)) - execution: Final = run_suite( - contract, - tmp_path, - ("full",), - python_inventory=_python_inventory, - rust_inventory=_rust_inventory, - ) - artifact: Final = MappingReportArtifact.model_validate_json(execution.artifacts[0].body) - - assert artifact.detailed diff --git a/tests/rust-python-harness/strategies/unit_tests_parity/__init__.py b/tests/rust-python-harness/strategies/unit_tests_parity/__init__.py index 0067bf6dfe5..fe3bd2e2f94 100644 --- a/tests/rust-python-harness/strategies/unit_tests_parity/__init__.py +++ b/tests/rust-python-harness/strategies/unit_tests_parity/__init__.py @@ -14,8 +14,8 @@ from ...shared.reporting.strategy import ( StrategyDefinition, SuiteCaseSpec, ) +from ...shared.unit_runners.contracts import UNIT_TEST_CONTRACTS from ...shared.unit_runners.suite_runner import run_suites -from ..unit_tests_mapping.mappings import UNIT_TEST_CONTRACTS from .reporting import render_unit_parity_results from .runner import UnitParityExclusion, UnitParitySuite, run_suite diff --git a/tests/rust-python-harness/strategies/unit_tests_rust/__init__.py b/tests/rust-python-harness/strategies/unit_tests_rust/__init__.py index 8114e12ab96..b9ca5b13e63 100644 --- a/tests/rust-python-harness/strategies/unit_tests_rust/__init__.py +++ b/tests/rust-python-harness/strategies/unit_tests_rust/__init__.py @@ -13,8 +13,8 @@ from ...shared.reporting.strategy import ( StrategyDefinition, SuiteCaseSpec, ) +from ...shared.unit_runners.contracts import UNIT_TEST_CONTRACTS from ...shared.unit_runners.suite_runner import run_suites -from ..unit_tests_mapping.mappings import UNIT_TEST_CONTRACTS from .reporting import render_rust_unit_results from .runner import RustSuite, run_suite diff --git a/tests/test_litellm/rust_bridge/native_route_wheel_test.py b/tests/test_litellm/rust_bridge/native_route_wheel_test.py index 8d83f4ca8a6..6f963cec6cc 100644 --- a/tests/test_litellm/rust_bridge/native_route_wheel_test.py +++ b/tests/test_litellm/rust_bridge/native_route_wheel_test.py @@ -283,8 +283,6 @@ async def exercise_async_concurrency(native: object, api_base: str) -> None: def exercise_routes(native_path: Path, api_base: str) -> object: native: Final = load_native(native_path) - if hasattr(native, "_trace"): - raise AssertionError("release wheel exposed trace-parity diagnostics") exercise_sync(native, api_base) asyncio.run(exercise_async(native, api_base)) asyncio.run(exercise_async_concurrency(native, api_base)) diff --git a/tests/test_rust_python_harness.py b/tests/test_rust_python_harness.py index 85b45c07bc2..a1bb370a074 100644 --- a/tests/test_rust_python_harness.py +++ b/tests/test_rust_python_harness.py @@ -1,8 +1,6 @@ from __future__ import annotations import importlib -from pathlib import Path -from types import SimpleNamespace from typing import Final import pytest @@ -10,16 +8,10 @@ import pytest models = importlib.import_module("tests.rust-python-harness.shared.reporting.models") strategy_module = importlib.import_module("tests.rust-python-harness.shared.reporting.strategy") ui = importlib.import_module("tests.rust-python-harness.shared.reporting.ui") -mapping_validator = importlib.import_module("tests.rust-python-harness.strategies.unit_tests_mapping.mapping_validator") -mappings = importlib.import_module("tests.rust-python-harness.strategies.unit_tests_mapping.mappings") -ocr_mapping = importlib.import_module("tests.rust-python-harness.strategies.unit_tests_mapping.cases.ocr") +contracts = importlib.import_module("tests.rust-python-harness.shared.unit_runners.contracts") cli = importlib.import_module("tests.rust-python-harness.cli") -native_build = importlib.import_module("tests.rust-python-harness.shared.native_build") -audit_mapping = mapping_validator.audit_mapping -UNIT_TEST_CONTRACTS = mappings.UNIT_TEST_CONTRACTS -OCR_CONTRACT = ocr_mapping.OCR_CONTRACT -REPO_ROOT = Path(__file__).resolve().parents[1] +UNIT_TEST_CONTRACTS = contracts.UNIT_TEST_CONTRACTS CaseResult = models.CaseResult Coverage = models.Coverage HarnessCase = models.HarnessCase @@ -49,7 +41,6 @@ def _case(module: str = "tests.example") -> HarnessCase: "tests.rust-python-harness.strategies.trace_parity.sdk.messages.case", "tests.rust-python-harness.strategies.trace_parity.sdk.chat_completions.case", "tests.rust-python-harness.strategies.trace_parity.sdk.transcription.case", - "tests.rust-python-harness.strategies.trace_parity.gateway.messages.case", ], ) def test_implemented_namespace_case_modules_remain_importable(module: str) -> None: @@ -117,70 +108,14 @@ def test_should_format_developer_facing_run_context() -> None: assert _format_duration(1.25) == "1.2s" -def test_should_leave_functions_without_mapping_contracts_unimplemented() -> None: +def test_should_leave_functions_without_unit_test_contracts_unimplemented() -> None: assert "messages" not in UNIT_TEST_CONTRACTS -def test_should_report_a_bridge_that_cannot_be_imported() -> None: - with pytest.MonkeyPatch.context() as patch: - patch.setattr(native_build, "get_native_bridge", lambda: None) - message: Final = native_build.trace_bridge_error() - - assert message is not None - assert "not importable" in message - - -def test_should_report_a_bridge_built_without_the_trace_feature() -> None: - with pytest.MonkeyPatch.context() as patch: - patch.setattr(native_build, "get_native_bridge", lambda: SimpleNamespace(_trace=None)) - message: Final = native_build.trace_bridge_error() - - assert message is not None - assert native_build.BRIDGE_FEATURE in message - - -def test_should_accept_a_bridge_built_with_the_trace_feature() -> None: - with pytest.MonkeyPatch.context() as patch: - patch.setattr(native_build, "get_native_bridge", lambda: SimpleNamespace(_trace=object())) - - assert native_build.trace_bridge_error() is None - - -def test_should_not_rebuild_the_bridge_while_reporting_its_state() -> None: - def forbidden_rebuild(repo_root: object) -> tuple[bool, str]: - raise AssertionError("trace_bridge_error must not rebuild the native bridge") - - with pytest.MonkeyPatch.context() as patch: - patch.setattr(native_build, "_rebuild", forbidden_rebuild) - patch.setattr(native_build, "get_native_bridge", lambda: None) - - assert native_build.trace_bridge_error() is not None - - -def test_should_derive_ocr_mapping_status_from_live_tests() -> None: - bridge_error: Final = native_build.trace_bridge_error() - if bridge_error is not None: - pytest.skip(bridge_error) - - report = audit_mapping(OCR_CONTRACT, repo_root=REPO_ROOT) - - assert report.is_valid, ( - f"Missing Python tests: {list(report.missing_python_tests)}\n" - f"Missing Rust tests: {list(report.missing_rust_tests)}\n" - f"Duplicate Python mappings: {list(report.duplicate_python_mappings)}\n" - f"Invalid mapping exclusions: {list(report.invalid_mapping_exclusions)}\n" - f"Invalid parity exclusions: {list(report.invalid_unit_parity_exclusions)}" - ) - assert report.mapped_count == len(OCR_CONTRACT.mapping.mappings) - assert report.total_count == ( - report.mapped_count + len(report.excluded_python_tests) + len(report.unmapped_python_tests) - ) - - def test_strategy_subcommand_accepts_function_filter(capsys: pytest.CaptureFixture[str]) -> None: - exit_code: Final = cli.main(["run", "unit_tests_mapping", "--function", "messages"]) + exit_code: Final = cli.main(["run", "unit_tests_rust", "--function", "messages"]) captured: Final = capsys.readouterr() assert exit_code == 0 assert "- messages: not_implemented" in captured.out - assert "unit_tests_mapping:messages: not_implemented" not in captured.out + assert "unit_tests_rust:messages: not_implemented" not in captured.out